From b192958cd87275055312c38fbdb6f18a6286012b Mon Sep 17 00:00:00 2001 From: Xuwznln <18435084+Xuwznln@users.noreply.github.com> Date: Sun, 19 Jul 2026 13:35:51 +0800 Subject: [PATCH 1/3] feat(action): add exception decision policies Route class-matched failures through backend approval and preserve normal, skipped, and operator-intervention outcomes. Co-authored-by: Cursor --- .../action_error_decision_frontend.md | 513 +++++++++++++ docs/developer_guide/http_api.md | 13 +- tests/test_action_policy.py | 726 ++++++++++++++++++ unilabos/app/model.py | 11 + unilabos/app/web/api.py | 115 ++- unilabos/app/web/controller.py | 37 + unilabos/app/web/event_bus.py | 115 +++ unilabos/app/ws_client.py | 51 ++ unilabos/registry/action_policy.py | 173 +++++ unilabos/registry/ast_registry_scanner.py | 7 +- unilabos/registry/decorators.py | 10 + unilabos/registry/registry.py | 4 + unilabos/ros/nodes/base_device_node.py | 57 +- unilabos/ros/nodes/presets/host_node.py | 722 +++++++++++++++-- unilabos/utils/type_check.py | 29 +- 15 files changed, 2519 insertions(+), 64 deletions(-) create mode 100644 docs/developer_guide/action_error_decision_frontend.md create mode 100644 tests/test_action_policy.py create mode 100644 unilabos/app/web/event_bus.py create mode 100644 unilabos/registry/action_policy.py diff --git a/docs/developer_guide/action_error_decision_frontend.md b/docs/developer_guide/action_error_decision_frontend.md new file mode 100644 index 000000000..e1c2b757b --- /dev/null +++ b/docs/developer_guide/action_error_decision_frontend.md @@ -0,0 +1,513 @@ +# 动作异常决策：前端接入协议 + +本文冻结动作执行失败后的前端接口与消息契约。接口风格参考 +`feat/edge-networking-and-scheduler` 的 Edge Monitor：REST 提供权威快照和命令入口，SSE +只提供实时增量；前端不维护调度权威，也不直接调用设备。 + +## 1. 职责边界 + +| 组件 | 职责 | +|---|---| +| 设备节点 | 执行动作；失败时返回 `suc:false` 和结构化 `error_info` | +| HostNode | 根据 Host 注册表解析策略；持有 pending、超时和重试次数；执行 retry/fallback/skip/abort | +| Host 微后端 | 为本地前端提供 pending 快照、SSE 增量和决策提交接口 | +| 云后端 | 为云端任务接收异常上报、承载用户交互并回传选择 | +| 前端 | 展示 Host/后端提供的选项并提交 `option.action`；不执行 fallback，不自行判定 job 终态 | + +决策通道由任务来源决定，不能交叉接管： + +- 从本地 `POST /api/v1/job/add` 创建的任务：`micro_backend`。 +- 从边云 WebSocket `job_start` 下发的任务：`backend`。 + +## 2. 完整数据流 + +### 2.1 Host 微后端模式 + +```text +Frontend Host microbackend HostNode Device + | POST /job/add | | send_goal | + |--------------------------->|------------------->|---------------------->| + | | | suc:false | + | | |<----------------------| + | | | registry 匹配策略 | + | | | 建立 pending + timer | + | SSE job_error_decision_required | | + |<------------------------------------------------| | + | GET /error-decisions | | | + |--------------------------->|------------------->| | + | POST /error-decisions/{id} | | | + |--------------------------->|------------------->| | + | SSE job_error_decision_resolved | | + |<------------------------------------------------| | + | | | retry/fallback goal | + | | |---------------------->| + | SSE job_status / GET /job/{job_id}/status | result | + |<------------------------------------------------|<----------------------| +``` + +### 2.2 云后端模式 + +```text +Device --suc:false--> HostNode --job_error_decision_required--> Cloud Backend +Device <--goal-------- HostNode <--job_error_decision----------- Cloud Backend +Cloud Backend <------------------job_status--------------------- HostNode +``` + +设备不会主动连接微后端或云后端，也不等待 HTTP/WebSocket 决策。等待状态只存在于 Host。 + +## 3. Host 微后端接口 + +默认地址为 `http://:8002`，OpenAPI 位于 `/api/docs`。 + +| 方法 | 路径 | 用途 | +|---|---|---| +| GET | `/api/v1/error-decisions` | 当前本地 pending 的权威列表 | +| POST | `/api/v1/error-decisions/{decision_id}` | 提交一次决策 | +| GET | `/api/v1/monitor/events?channels=action&backlog=40` | SSE 实时增量与有限回放 | +| GET | `/api/v1/monitor/snapshot` | 初始化及 SSE 丢事件后的权威快照 | +| GET | `/api/v1/job/{job_id}/status` | 查询原 job 当前状态或最终结果 | + +## 4. 异常报告结构 + +`job_error_decision_required` 和 `GET /error-decisions` 使用同一结构： + +```json +{ + "decision_id": "8a714f4c-5bb0-47b7-9245-9ddf907ef8d4", + "job_id": "df958dcb-b2bf-4a48-94a2-81410bf95a6b", + "task_id": "3f39b087-aec2-4b76-b31d-a3da277e7ec1", + "device_id": "pump-1", + "action_name": "transfer", + "exception_type": "CommunicationError", + "category": "network", + "severity": "error", + "error_message": "serial port closed", + "traceback": "Traceback ...", + "options": [ + { + "action": "retry", + "label": "重试" + }, + { + "action": "reset_connection", + "label": "重置连接", + "description": "重置设备连接后结束本次人工干预", + "fallback_action": { + "action_name": "reset", + "params": {"channel": 2} + } + }, + { + "action": "skip", + "label": "跳过" + }, + { + "action": "abort", + "label": "终止" + } + ], + "retry_count": 0, + "max_retries": 2, + "created_at": 1786440000.0, + "decision_timeout_seconds": 300.0, + "expires_at": 1786440300.0, + "default_on_decision_timeout": "abort", + "require_confirmation": true +} +``` + +字段规范： + +| 字段 | 必需 | 前端含义 | +|---|---:|---| +| `decision_id` | 是 | 决策唯一键；POST 路径参数 | +| `job_id` / `task_id` | 是 | 关联原任务；最终状态仍按原 `job_id` 查询 | +| `device_id` / `action_name` | 是 | 展示和日志定位，不作为前端执行地址 | +| `exception_type` | 是 | 异常类名 | +| `category` / `severity` | 否 | 设备异常提供时透传 | +| `error_message` | 是 | 面向用户的简要错误 | +| `traceback` | 是 | 调试详情；默认折叠，不建议直接 toast 全文 | +| `options` | 是 | Host 从注册表匹配出的唯一合法选择集合 | +| `retry_count` / `max_retries` | 是 | 当前已重试次数和上限 | +| `created_at` / `expires_at` | 是 | Unix 秒；用于展示倒计时 | +| `default_on_decision_timeout` | 是 | 到期后 Host 自动执行的动作 | + +前端必须以 `option.action` 为稳定值，`label/description` 只用于展示。 +`fallback_action` 是只读说明，浏览器不得调用其中的设备动作或修改参数。 + +## 5. REST 示例 + +### 5.1 初始化或断线恢复 + +```http +GET /api/v1/error-decisions HTTP/1.1 +Accept: application/json +``` + +```json +{ + "decisions": [ + { + "decision_id": "8a714f4c-5bb0-47b7-9245-9ddf907ef8d4", + "job_id": "df958dcb-b2bf-4a48-94a2-81410bf95a6b", + "device_id": "pump-1", + "action_name": "transfer", + "exception_type": "CommunicationError", + "error_message": "serial port closed", + "options": [{"action": "retry", "label": "重试"}], + "retry_count": 0, + "max_retries": 2, + "created_at": 1786440000.0, + "expires_at": 1786440300.0, + "decision_timeout_seconds": 300.0, + "default_on_decision_timeout": "abort", + "require_confirmation": true, + "traceback": "Traceback ...", + "task_id": "3f39b087-aec2-4b76-b31d-a3da277e7ec1" + } + ] +} +``` + +### 5.2 提交 retry/skip/abort + +```http +POST /api/v1/error-decisions/8a714f4c-5bb0-47b7-9245-9ddf907ef8d4 +Content-Type: application/json + +{"action":"retry","reason":"operator confirmed"} +``` + +成功只表示 Host 接受了命令，不表示恢复动作已经成功： + +```json +{ + "decision_id": "8a714f4c-5bb0-47b7-9245-9ddf907ef8d4", + "status": "delivered" +} +``` + +提交注册表自定义选项时，仍只传稳定 action： + +```json +{ + "action": "reset_connection", + "reason": "operator selected registered recovery" +} +``` + +如果选项要求人工给出替代结果，可附加 `result`： + +```json +{ + "action": "manual_result", + "result": {"confirmed_volume": 10.0}, + "reason": "verified on instrument" +} +``` + +错误语义： + +- `404`：不存在、已被其他请求处理、已经超时，或通道来源不匹配。 +- `503`：HostNode 尚未就绪。 +- 第一次合法决策获胜；前端收到 `404` 时重新 GET 列表。若列表中已不存在该 ID，关闭弹窗并继续追踪原 job。 + +### 5.3 查询原 job + +```http +GET /api/v1/job/df958dcb-b2bf-4a48-94a2-81410bf95a6b/status +``` + +等待决策、retry 或 fallback 执行期间，状态保持 `2`： + +```json +{ + "code": 0, + "data": { + "jobId": "df958dcb-b2bf-4a48-94a2-81410bf95a6b", + "status": 2, + "result": {} + }, + "message": "success" +} +``` + +状态码：`0 UNKNOWN`、`1 ACCEPTED`、`2 EXECUTING`、`3 CANCELING`、 +`4 SUCCEEDED`、`5 CANCELED`、`6 ABORTED`。 + +成功终态的 `result.suc_type`： + +- `normal`：原动作或 retry 正常成功。 +- `skip`：人工选择跳过；调度可继续，但物料侧应进入复核/隔离流程。 +- `operator_intervention`：fallback 或人工替代结果成功。 + +## 6. SSE 事件流 + +连接： + +```text +GET /api/v1/monitor/events?channels=action&backlog=40 +Accept: text/event-stream +``` + +每一帧与参考分支 MonitorBus 一致： + +```text +id: 17 +event: action +data: {"seq":17,"ts":1786440000.0,"channel":"action","type":"job_error_decision_required","data":{...report...},"trace_id":"","span_id":""} + +``` + +当前 action 事件类型： + +| `type` | `data` | 前端动作 | +|---|---|---| +| `job_error_decision_required` | 完整异常报告 | 按 `decision_id` upsert 弹窗/通知 | +| `job_error_decision_resolved` | ID、job、选择、原因、时间 | 移除 pending，锁定本次操作 | +| `job_status` | 与边云 `job_status.data` 同形状 | 更新 job 的 running/success/failed 投影 | + +`job_error_decision_resolved.data` 示例： + +```json +{ + "decision_id": "8a714f4c-5bb0-47b7-9245-9ddf907ef8d4", + "job_id": "df958dcb-b2bf-4a48-94a2-81410bf95a6b", + "task_id": "3f39b087-aec2-4b76-b31d-a3da277e7ec1", + "device_id": "pump-1", + "action_name": "transfer", + "selected_action": "retry", + "reason": "operator confirmed", + "resolved_at": 1786440020.0 +} +``` + +SSE 是增量通知，不是权威数据库： + +1. 页面启动先 GET `/monitor/snapshot` 或 `/error-decisions`。 +2. 再建立 EventSource，并用 `addEventListener("action", ...)` 接收命名事件。 +3. 保存最近 `seq`；忽略 `seq <= lastSeq` 的回放重复帧。发现向前跳号、浏览器重连或页面恢复可见时，重新 GET snapshot。 +4. 慢消费者可能丢事件，Host 执行不会被 SSE 反压。 +5. SSE 约每 15 秒发送 keepalive，浏览器按 `retry: 3000` 自动重连。 + +Snapshot 示例： + +```json +{ + "now": 1786440010.0, + "host_ready": true, + "pending_error_decisions": [], + "recent": { + "action": [] + } +} +``` + +## 7. TypeScript 接入示例 + +```ts +type ErrorOption = { + action: string; + label: string; + description?: string; + fallback_action?: { + action_name: string; + params?: Record; + }; +}; + +type ErrorDecision = { + decision_id: string; + job_id: string; + task_id: string; + device_id: string; + action_name: string; + exception_type: string; + category?: string; + severity?: string; + error_message: string; + traceback: string; + options: ErrorOption[]; + retry_count: number; + max_retries: number; + created_at: number; + expires_at: number; + decision_timeout_seconds: number; + default_on_decision_timeout: "abort" | "retry" | "skip"; + require_confirmation: true; +}; + +type MonitorEvent = { + seq: number; + ts: number; + channel: "action"; + type: + | "job_error_decision_required" + | "job_error_decision_resolved" + | "job_status"; + data: Record; + trace_id: string; + span_id: string; +}; + +const baseUrl = "http://127.0.0.1:8002"; +const pending = new Map(); +let lastSeq = 0; + +async function refreshDecisions() { + const response = await fetch(`${baseUrl}/api/v1/error-decisions`); + if (!response.ok) throw new Error(`decision snapshot ${response.status}`); + const body = (await response.json()) as { decisions: ErrorDecision[] }; + pending.clear(); + for (const decision of body.decisions) { + pending.set(decision.decision_id, decision); + } + renderDecisionCenter([...pending.values()]); +} + +function connectMonitor() { + const source = new EventSource( + `${baseUrl}/api/v1/monitor/events?channels=action&backlog=40`, + ); + + source.addEventListener("action", async (message) => { + const event = JSON.parse((message as MessageEvent).data) as MonitorEvent; + if (lastSeq !== 0 && event.seq <= lastSeq) return; + if (lastSeq !== 0 && event.seq > lastSeq + 1) { + await refreshDecisions(); + } + lastSeq = event.seq; + + if (event.type === "job_error_decision_required") { + const decision = event.data as unknown as ErrorDecision; + pending.set(decision.decision_id, decision); + renderDecisionCenter([...pending.values()]); + } else if (event.type === "job_error_decision_resolved") { + pending.delete(String(event.data.decision_id)); + renderDecisionCenter([...pending.values()]); + } else if (event.type === "job_status") { + updateJobProjection(event.data); + } + }); + + source.onerror = () => { + // EventSource 会自动重连；恢复后仍应以 REST snapshot 校准。 + // 服务进程重启时 seq 会从 1 重新开始，因此不能沿用旧连接的 lastSeq。 + lastSeq = 0; + void refreshDecisions(); + }; + return source; +} + +async function resolveDecision( + decision: ErrorDecision, + action: string, + result?: unknown, +) { + if (!decision.options.some((option) => option.action === action)) { + throw new Error("option is not allowed by Host registry policy"); + } + const response = await fetch( + `${baseUrl}/api/v1/error-decisions/${decision.decision_id}`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + action, + ...(result === undefined ? {} : { result }), + reason: "operator confirmed", + }), + }, + ); + if (response.status === 404) { + await refreshDecisions(); + return; + } + if (!response.ok) throw new Error(`resolve decision ${response.status}`); +} + +void refreshDecisions().then(connectMonitor); +``` + +`renderDecisionCenter` 和 `updateJobProjection` 是前端自己的 store/UI 适配点。 + +## 8. 云后端 WebSocket 契约 + +Host → Backend： + +```json +{ + "action": "job_error_decision_required", + "data": { + "decision_id": "8a714f4c-5bb0-47b7-9245-9ddf907ef8d4", + "job_id": "df958dcb-b2bf-4a48-94a2-81410bf95a6b", + "task_id": "3f39b087-aec2-4b76-b31d-a3da277e7ec1", + "device_id": "pump-1", + "action_name": "transfer", + "exception_type": "CommunicationError", + "error_message": "serial port closed", + "options": [{"action": "retry", "label": "重试"}], + "retry_count": 0, + "max_retries": 2, + "created_at": 1786440000.0, + "expires_at": 1786440300.0, + "decision_timeout_seconds": 300.0, + "default_on_decision_timeout": "abort", + "require_confirmation": true, + "traceback": "Traceback ..." + } +} +``` + +Backend → Host： + +```json +{ + "action": "job_error_decision", + "data": { + "decision_id": "8a714f4c-5bb0-47b7-9245-9ddf907ef8d4", + "job_id": "df958dcb-b2bf-4a48-94a2-81410bf95a6b", + "device_id": "pump-1", + "action": "retry", + "reason": "operator confirmed" + } +} +``` + +云前端不应直接连接 Host 的异常决策 REST；它只消费云后端持久投影并向云后端提交选择。 +云后端必须原样保留 `decision_id/job_id/device_id`，回包时三者共同校验。 + +## 9. 前端状态机 + +```text +absent + └─ required/snapshot ─> pending + ├─ POST 中 ─> resolving（按钮禁用） + ├─ resolved ─> tracking_job + └─ expires_at 到达 ─> refresh snapshot +tracking_job + ├─ job status=2 ─> tracking_job + └─ job status=4/5/6 ─> terminal +``` + +关键不变量： + +1. `decision_id` 是弹窗/通知的唯一键，不能用 `device_id` 去重。 +2. pending 期间原 job 仍为执行中，不能先标记 failed。 +3. POST `delivered` 不是 job 成功，只是 Host 已接受选择。 +4. retry 使用新的 ROS transport goal UUID，但前端始终追踪原 `job_id`。 +5. 只允许提交报告 `options` 中的 action；最终合法性仍由 Host 校验。 +6. Host 超时是权威；浏览器倒计时归零后只刷新，不自行执行默认动作。 +7. fallback 由 Host 通过 ActionClient 发给真实设备，浏览器绝不调用设备 Service/Action。 + +## 10. 前端验收清单 + +- 页面刷新后能通过 REST 恢复已有 pending。 +- 新异常通过 SSE 在不刷新页面时出现。 +- 同一 `decision_id` 的 snapshot/SSE 重复消息只产生一个 UI 项。 +- 点击后立即禁用按钮，成功响应后继续追踪原 job。 +- POST 响应丢失后再次提交得到 404，前端通过 snapshot 正确收敛。 +- SSE `seq` 出现空洞时重新拉 snapshot。 +- timeout、另一个浏览器先处理、云/本地通道错投时不会重复执行。 +- retry/fallback 成功后展示 `suc_type`；skip 明确提示需要物料复核。 +- traceback 默认折叠，错误摘要、设备、动作、重试次数和倒计时默认可见。 diff --git a/docs/developer_guide/http_api.md b/docs/developer_guide/http_api.md index a1f548df6..e2905d41d 100644 --- a/docs/developer_guide/http_api.md +++ b/docs/developer_guide/http_api.md @@ -204,7 +204,7 @@ curl -X GET "http://localhost:8002/api/v1/job/b6acb586-733a-42ab-9f73-55c9a52aa8 } ``` -> **注意**: 任务结果在首次查询后会被自动删除，请确保保存返回的结果数据。 +> **注意**: 任务状态和结果可重复查询，不会因前端第一次读取而删除。微后端仍会按任务结果存储的清理策略回收过期记录。 ## API 端点列表 @@ -225,6 +225,17 @@ curl -X GET "http://localhost:8002/api/v1/job/b6acb586-733a-42ab-9f73-55c9a52aa8 | `/api/v1/job/add` | POST | 提交新任务 | | `/api/v1/job/{job_id}/status` | GET | 查询任务状态和结果 | +### 动作异常决策相关 + +| 端点 | 方法 | 说明 | +| --------------------------------------------------- | ---- | -------------------------------- | +| `/api/v1/error-decisions` | GET | 获取尚未处理的动作异常决策 | +| `/api/v1/error-decisions/{decision_id}` | POST | 提交一项动作异常决策 | +| `/api/v1/monitor/events` | GET | 订阅动作状态与异常决策 SSE 事件 | +| `/api/v1/monitor/snapshot` | GET | 获取异常决策及近期事件权威快照 | + +这组接口直接返回业务 JSON，HTTP 错误使用 FastAPI 的 `detail` 结构，不套用本页其他接口的 `code/data/message` 外层。完整字段、SSE 事件、TypeScript 示例和前端状态机见[动作异常决策：前端接入协议](action_error_decision_frontend.md)。 + ### 资源相关 | 端点 | 方法 | 说明 | diff --git a/tests/test_action_policy.py b/tests/test_action_policy.py new file mode 100644 index 000000000..0a26aea67 --- /dev/null +++ b/tests/test_action_policy.py @@ -0,0 +1,726 @@ +import asyncio +import ast +import json + +import pytest + +from unilabos.app.ws_client import MessageProcessor, QueueItem +from unilabos.registry.action_policy import ( + ERROR_DECISION_TARGET_BACKEND, + ERROR_DECISION_TARGET_MICRO_BACKEND, + SUCCESS_TYPE_NORMAL, + SUCCESS_TYPE_OPERATOR_INTERVENTION, + SUCCESS_TYPE_SKIP, + normalize_error_policy, + resolve_error_options, +) +from unilabos.registry.ast_registry_scanner import ( + _collect_imports, + _extract_class_body, +) +from unilabos.registry.decorators import action, get_action_meta +from unilabos.ros.nodes.presets.host_node import HostNode +from unilabos.utils.type_check import ( + get_result_info_str, + serialize_result_info, +) + + +class CommunicationError(Exception): + pass + + +class ModbusCommunicationError(CommunicationError): + pass + + +def _policy(): + return { + "options": { + "CommunicationError": [ + {"action": "retry", "label": "重试"}, + { + "action": "reset_connection", + "label": "审批后重置连接", + "fallback_action": { + "action_name": "reset", + "params": {"channel": 2}, + }, + }, + ], + "*": [{"action": "abort", "label": "终止"}], + }, + "max_retries": 2, + "decision_timeout_seconds": 30, + } + + +def test_policy_matches_exception_mro_and_preserves_server_action(): + policy = normalize_error_policy(_policy()) + + options = resolve_error_options( + policy, + ModbusCommunicationError("offline"), + ) + + assert [option["action"] for option in options] == [ + "retry", + "reset_connection", + ] + assert options[1]["fallback_action"] == { + "action_name": "reset", + "params": {"channel": 2}, + } + + +def test_policy_uses_wildcard_for_unmatched_exception(): + policy = normalize_error_policy(_policy()) + + assert resolve_error_options(policy, ValueError("bad")) == [ + {"action": "abort", "label": "终止"} + ] + + +def test_policy_accepts_legacy_fallback_action_string(): + policy = normalize_error_policy( + { + "options": { + "ValueError": [ + { + "action": "reset", + "label": "重置", + "fallback_action": "reset_device", + } + ] + } + } + ) + + assert policy["options"]["ValueError"][0]["fallback_action"] == { + "action_name": "reset_device", + "params": {}, + } + + +def test_action_exposes_normalized_policy_in_runtime_and_registry_meta(): + @action(error_policy=_policy()) + def run(self): + return None + + assert run._action_error_policy == get_action_meta(run)["error_policy"] + assert run._action_error_policy["options"]["CommunicationError"][1][ + "fallback_action" + ]["params"] == {"channel": 2} + + +def test_ast_scanner_preserves_exception_class_option_mapping(): + source = """ +from unilabos.registry.decorators import action + +class Driver: + @action(error_policy={ + "options": { + "ValueError": [ + { + "action": "inspect", + "label": "人工检查", + "fallback_action": { + "action_name": "inspect_device", + "params": {"station": "A"}, + }, + } + ] + } + }) + def run(self): + pass +""" + tree = ast.parse(source) + class_node = next( + node for node in tree.body if isinstance(node, ast.ClassDef) + ) + extracted = _extract_class_body(class_node, _collect_imports(tree)) + + value_error_options = extracted["actions"]["run"]["action_args"][ + "error_policy" + ]["options"]["ValueError"] + assert value_error_options[0]["fallback_action"]["params"] == { + "station": "A" + } + + +@pytest.mark.parametrize( + ("suc_type", "return_value"), + [ + (SUCCESS_TYPE_NORMAL, {"value": 1}), + (SUCCESS_TYPE_SKIP, None), + (SUCCESS_TYPE_OPERATOR_INTERVENTION, {"recovered": True}), + ], +) +def test_result_info_distinguishes_three_success_types(suc_type, return_value): + encoded = json.loads( + get_result_info_str("", True, return_value, suc_type=suc_type) + ) + serialized = serialize_result_info( + "", + True, + return_value, + suc_type=suc_type, + ) + + assert encoded == serialized + assert encoded["suc"] is True + assert encoded["suc_type"] == suc_type + assert encoded["return_value"] == return_value + + +def test_failed_result_does_not_claim_success_type(): + result = serialize_result_info("failed", False, None) + + assert result == {"error": "failed", "suc": False, "return_value": None} + + +def test_policy_rejects_empty_class_options(): + with pytest.raises(ValueError, match="非空列表"): + normalize_error_policy({"options": {"ValueError": []}}) + + +def test_policy_rejects_duplicate_actions_for_one_exception(): + with pytest.raises(ValueError, match="重复 action"): + normalize_error_policy( + { + "options": { + "ValueError": [ + {"action": "retry", "label": "重试一次"}, + {"action": "retry", "label": "再次重试"}, + ] + } + } + ) + + +@pytest.mark.parametrize( + "field,value", + [ + ("max_retries", True), + ("decision_timeout_seconds", False), + ], +) +def test_policy_rejects_boolean_numeric_settings(field, value): + policy = { + "options": {"ValueError": [{"action": "abort", "label": "终止"}]}, + field: value, + } + with pytest.raises(ValueError): + normalize_error_policy(policy) + + +def test_failed_result_carries_structured_error_info(): + error_info = { + "exception_type": "CommunicationError", + "exception_mro": ["CommunicationError", "Exception"], + } + + encoded = json.loads( + get_result_info_str("offline", False, None, error_info=error_info) + ) + serialized = serialize_result_info( + "offline", + False, + None, + error_info=error_info, + ) + + assert encoded == serialized + assert encoded["error_info"] == error_info + assert "suc_type" not in encoded + + +class _Logger: + def info(self, message): + pass + + def warning(self, message): + pass + + +class _DecisionBridge: + def __init__(self): + self.reports = [] + + def publish_job_error_decision_required(self, report): + self.reports.append(report) + return True + + +class FakeHostDecisionNode: + _begin_action_error_decision = HostNode._begin_action_error_decision + _emit_local_action_event = staticmethod(HostNode._emit_local_action_event) + _handle_action_error_decision_timeout = ( + HostNode._handle_action_error_decision_timeout + ) + handle_action_error_decision = HostNode.handle_action_error_decision + get_pending_action_error_decisions = ( + HostNode.get_pending_action_error_decisions + ) + + def __init__(self): + import threading + + self.bridge = _DecisionBridge() + self.bridges = [self.bridge] + self._goals = {"job-1": object()} + self._pending_action_error_decisions = {} + self._pending_action_error_decisions_lock = threading.RLock() + self._error_execution_contexts = { + "job-1": { + "item": _queue_item(), + "action_type": "UniLabJsonCommand", + "action_kwargs": {"channel": 1}, + "sample_material": {}, + "server_info": None, + "retry_count": 0, + } + } + self._action_value_mappings = { + "device-1": { + "run": { + "type": "UniLabJsonCommand", + "error_policy": normalize_error_policy( + { + "options": { + "CommunicationError": [ + {"action": "retry", "label": "重试"}, + {"action": "skip", "label": "跳过"}, + {"action": "abort", "label": "终止"}, + ] + }, + "max_retries": 2, + "decision_timeout_seconds": 30, + } + ), + }, + "auto-reset": {"type": "UniLabJsonCommand"}, + } + } + self.sent_goals = [] + self.finished = [] + + def lab_logger(self): + return _Logger() + + def send_goal(self, *args, **kwargs): + self.sent_goals.append((args, kwargs)) + + def _finish_error_handled_job( + self, + item, + status, + return_info, + result_data, + ): + self.finished.append((item, status, return_info, result_data)) + self._error_execution_contexts.pop(item.job_id, None) + + +def _queue_item( + error_decision_target=ERROR_DECISION_TARGET_BACKEND, +): + return QueueItem( + task_type="job_call_back_status", + device_id="device-1", + action_name="run", + task_id="task-1", + job_id="job-1", + notebook_id="notebook-1", + device_action_key="/devices/device-1/run", + error_decision_target=error_decision_target, + ) + + +def _error_return_info(): + return serialize_result_info( + "offline", + False, + None, + error_info={ + "action_name": "run", + "exception_type": "CommunicationError", + "exception_mro": [ + "CommunicationError", + "Exception", + "BaseException", + "object", + ], + "error_message": "offline", + "traceback": "trace", + }, + ) + + +def _begin_pending(host, policy=None, item=None): + if policy is not None: + host._action_value_mappings["device-1"]["run"]["error_policy"] = ( + normalize_error_policy(policy) + ) + pending_item = item or _queue_item() + host._error_execution_contexts["job-1"]["item"] = pending_item + assert host._begin_action_error_decision( + pending_item, + _error_return_info(), + {"return_info": "failed"}, + ) + decision_id = next(iter(host._pending_action_error_decisions)) + return decision_id + + +def test_host_owns_decision_and_publishes_registry_options(): + host = FakeHostDecisionNode() + decision_id = _begin_pending(host) + + report = host.bridge.reports[0] + assert report["decision_id"] == decision_id + assert report["device_id"] == "device-1" + assert report["exception_type"] == "CommunicationError" + assert [option["action"] for option in report["options"]] == [ + "retry", + "skip", + "abort", + ] + assert report["expires_at"] > report["created_at"] + assert report["max_retries"] == 2 + assert report["default_on_decision_timeout"] == "abort" + assert "job-1" not in host._goals + + assert host.handle_action_error_decision( + decision_id, + "job-1", + {"action": "abort"}, + ) + + +def test_ws_decision_routes_to_host_not_device(monkeypatch): + received = [] + + class _Host: + def handle_action_error_decision( + self, + decision_id, + job_id, + decision, + *, + decision_target=None, + ): + received.append((decision_id, job_id, decision, decision_target)) + return True + + monkeypatch.setattr( + HostNode, + "get_instance", + classmethod(lambda cls, index=0: _Host()), + ) + payload = { + "decision_id": "decision-ws", + "job_id": "job-ws", + "device_id": "remote-device", + "action": "retry", + } + + asyncio.run(MessageProcessor._handle_job_error_decision(object(), payload)) + + assert received == [ + ( + "decision-ws", + "job-ws", + payload, + ERROR_DECISION_TARGET_BACKEND, + ) + ] + + +def test_host_micro_backend_decision_stays_local_and_rejects_cloud_reply(): + from unilabos.app.web.event_bus import monitor_bus + + sub_id, event_queue, _ = monitor_bus.subscribe(channels={"action"}) + host = FakeHostDecisionNode() + try: + decision_id = _begin_pending( + host, + item=_queue_item(ERROR_DECISION_TARGET_MICRO_BACKEND), + ) + + required_event = event_queue.get(timeout=1) + assert required_event["channel"] == "action" + assert required_event["type"] == "job_error_decision_required" + assert required_event["data"]["decision_id"] == decision_id + assert required_event["data"]["expires_at"] > required_event["data"]["created_at"] + assert host.bridge.reports == [] + reports = host.get_pending_action_error_decisions( + ERROR_DECISION_TARGET_MICRO_BACKEND, + ) + assert [report["decision_id"] for report in reports] == [decision_id] + assert not host.handle_action_error_decision( + decision_id, + "job-1", + {"action": "abort"}, + decision_target=ERROR_DECISION_TARGET_BACKEND, + ) + assert host.handle_action_error_decision( + decision_id, + "job-1", + {"action": "retry"}, + decision_target=ERROR_DECISION_TARGET_MICRO_BACKEND, + ) + resolved_event = event_queue.get(timeout=1) + assert resolved_event["type"] == "job_error_decision_resolved" + assert resolved_event["data"]["selected_action"] == "retry" + assert ( + host.sent_goals[0][0][0].error_decision_target + == ERROR_DECISION_TARGET_MICRO_BACKEND + ) + finally: + monitor_bus.unsubscribe(sub_id) + + +def test_local_api_job_targets_host_micro_backend(monkeypatch): + from unilabos.app.model import JobAddReq + from unilabos.app.web import controller + + sent = [] + + class _Host: + def send_goal(self, item, *args, **kwargs): + sent.append(item) + + monkeypatch.setattr( + HostNode, + "get_instance", + classmethod(lambda cls, index=0: _Host()), + ) + monkeypatch.setattr( + controller, + "_get_action_type", + lambda device_id, action_name: "UniLabJsonCommand", + ) + monkeypatch.setattr( + controller, + "check_device_action_busy", + lambda device_id, action_name: (False, None), + ) + + result = controller.job_add( + JobAddReq( + device_id="device-1", + action="run", + sample_material={}, + ) + ) + + assert result.status == 1 + assert sent[0].error_decision_target == ERROR_DECISION_TARGET_MICRO_BACKEND + + +def test_micro_backend_rest_contract_roundtrip(monkeypatch): + from fastapi import FastAPI + from fastapi.testclient import TestClient + + from unilabos.app.web.api import api + from unilabos.app.web.controller import job_result_store, store_job_result + + host = FakeHostDecisionNode() + decision_id = _begin_pending( + host, + item=_queue_item(ERROR_DECISION_TARGET_MICRO_BACKEND), + ) + monkeypatch.setattr( + HostNode, + "get_instance", + classmethod(lambda cls, index=0: host), + ) + app = FastAPI() + app.include_router(api, prefix="/api/v1") + client = TestClient(app) + + paths = app.openapi()["paths"] + assert "/api/v1/error-decisions" in paths + assert "/api/v1/error-decisions/{decision_id}" in paths + assert "/api/v1/monitor/events" in paths + assert "/api/v1/monitor/snapshot" in paths + + response = client.get("/api/v1/error-decisions") + assert response.status_code == 200 + assert response.json()["decisions"][0]["decision_id"] == decision_id + snapshot = client.get("/api/v1/monitor/snapshot") + assert snapshot.status_code == 200 + assert snapshot.json()["host_ready"] is True + assert snapshot.json()["pending_error_decisions"][0]["decision_id"] == decision_id + + response = client.post( + f"/api/v1/error-decisions/{decision_id}", + json={"action": "skip", "reason": "operator confirmed"}, + ) + assert response.status_code == 200 + assert response.json() == {"decision_id": decision_id, "status": "delivered"} + + response = client.post( + f"/api/v1/error-decisions/{decision_id}", + json={"action": "skip"}, + ) + assert response.status_code == 404 + + store_job_result( + "job-poll", + "success", + {"suc": True, "suc_type": "skip", "return_value": None}, + ) + try: + first = client.get("/api/v1/job/job-poll/status") + second = client.get("/api/v1/job/job-poll/status") + assert first.status_code == second.status_code == 200 + assert first.json()["data"] == second.json()["data"] + assert first.json()["data"]["status"] == 4 + finally: + job_result_store.get_and_remove("job-poll") + + +def test_monitor_bus_sse_contract_and_bounded_replay(): + from unilabos.app.web.event_bus import MonitorBus, format_sse_event + + bus = MonitorBus(history=2) + bus.emit("action", "job_status", {"job_id": "job-1", "status": "running"}) + bus.emit("action", "job_status", {"job_id": "job-1", "status": "success"}) + bus.emit("action", "job_status", {"job_id": "job-2", "status": "failed"}) + + sub_id, _, replay = bus.subscribe(channels={"action"}, backlog=10) + try: + assert [event["seq"] for event in replay] == [2, 3] + encoded = format_sse_event(replay[-1]) + assert encoded.startswith("id: 3\nevent: action\ndata: ") + assert '"job_id": "job-2"' in encoded + assert encoded.endswith("\n\n") + finally: + bus.unsubscribe(sub_id) + + +def test_host_retry_uses_existing_action_client_path_and_new_transport_id(): + host = FakeHostDecisionNode() + decision_id = _begin_pending(host) + + assert host.handle_action_error_decision( + decision_id, + "job-1", + {"action": "retry"}, + ) + + args, kwargs = host.sent_goals[0] + assert args[0].job_id == "job-1" + assert args[1] == "UniLabJsonCommand" + assert args[2] == {"channel": 1} + assert kwargs["cache_error_context"] is False + assert kwargs["transport_goal_id"] != "job-1" + assert host._error_execution_contexts["job-1"]["retry_count"] == 1 + assert HostNode.get_goal_status(host, "job-1") == 2 + assert not host.finished + + +def test_host_decision_validates_identity_and_first_result_wins(): + host = FakeHostDecisionNode() + decision_id = _begin_pending(host) + + assert not host.handle_action_error_decision( + decision_id, + "other-job", + {"action": "retry"}, + ) + assert not host.handle_action_error_decision( + decision_id, + "job-1", + {"decision_id": "other-decision", "action": "retry"}, + ) + assert host.handle_action_error_decision( + decision_id, + "job-1", + {"action": "skip", "result": {"ignored": True}}, + ) + assert not host.handle_action_error_decision( + decision_id, + "job-1", + {"action": "abort"}, + ) + assert host.finished[0][1] == "success" + assert host.finished[0][2]["suc_type"] == SUCCESS_TYPE_SKIP + + +def test_host_retry_limit_fails_closed(): + host = FakeHostDecisionNode() + host._error_execution_contexts["job-1"]["retry_count"] = 1 + decision_id = _begin_pending( + host, + { + "options": { + "CommunicationError": [ + {"action": "retry", "label": "重试"} + ] + }, + "max_retries": 1, + }, + ) + + assert host.handle_action_error_decision( + decision_id, + "job-1", + {"action": "retry"}, + ) + + assert not host.sent_goals + assert host.finished[0][1] == "failed" + assert "exceeded 1 retries" in host.finished[0][2]["error"] + + +def test_host_dispatches_registered_fallback_action(): + host = FakeHostDecisionNode() + options = [ + { + "action": "reset_connection", + "label": "重置连接", + "fallback_action": { + "action_name": "reset", + "params": {"channel": 2}, + }, + } + ] + decision_id = _begin_pending( + host, + {"options": {"CommunicationError": options}}, + ) + + assert host.handle_action_error_decision( + decision_id, + "job-1", + {"action": "reset_connection"}, + ) + + args, kwargs = host.sent_goals[0] + assert args[0].action_name == "auto-reset" + assert args[2] == {"channel": 2} + assert kwargs["result_item"].action_name == "run" + assert kwargs["recovery_suc_type"] == SUCCESS_TYPE_OPERATOR_INTERVENTION + assert kwargs["cache_error_context"] is False + + +def test_host_rejects_unconfigured_backend_option_without_consuming_pending(): + host = FakeHostDecisionNode() + decision_id = _begin_pending(host) + + assert not host.handle_action_error_decision( + decision_id, + "job-1", + {"action": "force_success"}, + ) + assert decision_id in host._pending_action_error_decisions + + assert host.handle_action_error_decision( + decision_id, + "job-1", + {"action": "abort"}, + ) diff --git a/unilabos/app/model.py b/unilabos/app/model.py index 3a031aaaf..448f4eae4 100644 --- a/unilabos/app/model.py +++ b/unilabos/app/model.py @@ -1,3 +1,5 @@ +from typing import Any + from pydantic import BaseModel, Field @@ -69,6 +71,15 @@ class JobAddReq(BaseModel): data: dict = Field(examples=[{"position": 30, "torque": 5, "action": "push_to"}], default_factory=dict) +class ErrorDecisionIn(BaseModel): + """Host 微后端提交的异常处理决策。""" + + action: str = "" + option: Any = None + result: Any = None + reason: str = "" + + class JobStepFinishReq(BaseModel): token: str = Field(examples=["030944"], description="token") request_time: str = Field(examples=["2024-12-12 12:12:12.xxx"], description="requestTime") diff --git a/unilabos/app/web/api.py b/unilabos/app/web/api.py index 99981f776..e8ae71291 100644 --- a/unilabos/app/web/api.py +++ b/unilabos/app/web/api.py @@ -4,10 +4,13 @@ 提供API路由和处理函数 """ -from fastapi import APIRouter, WebSocket, WebSocketDisconnect +from fastapi import APIRouter, HTTPException, Request, WebSocket, WebSocketDisconnect import asyncio +import queue as queue_mod +import time import yaml +from fastapi.responses import StreamingResponse from unilabos.app.web.controller import ( devices, @@ -17,6 +20,8 @@ get_device_actions, get_action_schema, get_all_available_actions, + get_pending_action_error_decisions, + submit_action_error_decision, ) from unilabos.app.model import ( Resp, @@ -24,6 +29,7 @@ JobStatusResp, JobAddResp, JobAddReq, + ErrorDecisionIn, JobData, ) from unilabos.app.web.utils.host_utils import get_host_node_info @@ -1302,10 +1308,115 @@ def api_get_all_actions(): return Resp(data=data) +@api.get("/error-decisions", summary="查询本地待处理的动作异常决策") +def api_get_pending_action_error_decisions(): + """查询由 Host 微后端负责处理的异常决策。""" + + isok, data = get_pending_action_error_decisions() + if not isok: + raise HTTPException( + status_code=503, + detail=data.get("error", "Host node not initialized"), + ) + return data + + +@api.post( + "/error-decisions/{decision_id}", + summary="提交本地动作异常处理决策", +) +def api_submit_action_error_decision(decision_id: str, req: ErrorDecisionIn): + """提交 retry/skip/abort 或注册表声明的 fallback 选项。""" + + if hasattr(req, "model_dump"): + decision = req.model_dump(exclude_unset=True) + else: # Pydantic v1 compatibility + decision = req.dict(exclude_unset=True) + isok, data = submit_action_error_decision(decision_id, decision) + if not isok: + raise HTTPException( + status_code=404, + detail=data.get("error", "Pending decision not found"), + ) + return data + + +@api.get("/monitor/events", summary="订阅 Host 微后端实时事件") +async def monitor_events( + request: Request, + channels: str = "", + backlog: int = 40, +) -> StreamingResponse: + """SSE 增量流；断线后由 EventSource 重连，并以 snapshot 校准。""" + + from unilabos.app.web.event_bus import ( + CHANNELS, + format_sse_event, + monitor_bus, + ) + + requested = {channel.strip() for channel in channels.split(",") if channel.strip()} + channel_filter = (requested & set(CHANNELS)) or None + sub_id, subscriber_queue, replay = monitor_bus.subscribe( + channels=channel_filter, + backlog=max(0, min(backlog, 200)), + ) + + async def stream(): + try: + active_channels = ",".join(sorted(channel_filter or set(CHANNELS))) + yield f"retry: 3000\n: connected channels={active_channels}\n\n" + for event in replay: + yield format_sse_event(event) + last_beat = time.time() + while True: + if await request.is_disconnected(): + break + sent = False + while True: + try: + event = subscriber_queue.get_nowait() + except queue_mod.Empty: + break + yield format_sse_event(event) + sent = True + if sent: + last_beat = time.time() + continue + if time.time() - last_beat > 15: + yield ": keepalive\n\n" + last_beat = time.time() + await asyncio.sleep(0.4) + finally: + monitor_bus.unsubscribe(sub_id) + + return StreamingResponse( + stream(), + media_type="text/event-stream", + headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"}, + ) + + +@api.get("/monitor/snapshot", summary="获取 Host 微后端监控快照") +def monitor_snapshot(): + """前端初始化与 SSE 丢事件后的权威快照。""" + + from unilabos.app.web.event_bus import monitor_bus + + host_ready, pending = get_pending_action_error_decisions() + return { + "now": time.time(), + "host_ready": host_ready, + "pending_error_decisions": pending.get("decisions", []), + "recent": {"action": monitor_bus.recent("action", 40)}, + } + + @api.get("/job/{id}/status", summary="Job status", response_model=JobStatusResp) def job_status(id: str): """获取任务状态""" - data = job_info(id) + # 前端轮询必须幂等；结果由 JobResultStore 的 TTL 统一清理。 + data = job_info(id, remove_after_read=False) return JobStatusResp(data=data) diff --git a/unilabos/app/web/controller.py b/unilabos/app/web/controller.py index 147b4d207..6f6d4136d 100644 --- a/unilabos/app/web/controller.py +++ b/unilabos/app/web/controller.py @@ -12,6 +12,7 @@ from typing import Optional, Dict, Any, Tuple from unilabos.app.model import JobAddReq, JobData +from unilabos.registry.action_policy import ERROR_DECISION_TARGET_MICRO_BACKEND from unilabos.ros.nodes.presets.host_node import HostNode from unilabos.utils import logger @@ -322,6 +323,7 @@ def job_add(req: JobAddReq) -> JobData: job_id=job_id, notebook_id=req.notebook_id, device_action_key=device_action_key, + error_decision_target=ERROR_DECISION_TARGET_MICRO_BACKEND, ) host_node.send_goal( @@ -347,6 +349,41 @@ def job_add(req: JobAddReq) -> JobData: return JobData(jobId=job_id, status=6) # ABORTED +def get_pending_action_error_decisions() -> Tuple[bool, Dict[str, Any]]: + """读取应由 Host 微后端处理的异常决策。""" + + host_node = HostNode.get_instance(0) + if host_node is None: + return False, {"error": "Host node not initialized"} + return True, { + "decisions": host_node.get_pending_action_error_decisions( + decision_target=ERROR_DECISION_TARGET_MICRO_BACKEND, + ) + } + + +def submit_action_error_decision( + decision_id: str, + decision: Dict[str, Any], +) -> Tuple[bool, Dict[str, Any]]: + """将 Host 微后端的选择提交给 HostNode。""" + + host_node = HostNode.get_instance(0) + if host_node is None: + return False, {"error": "Host node not initialized"} + decision_id = str(decision_id or "") + if not decision_id: + return False, {"error": "decision_id is required"} + if not host_node.handle_action_error_decision( + decision_id, + "", + decision, + decision_target=ERROR_DECISION_TARGET_MICRO_BACKEND, + ): + return False, {"error": "pending action error decision not found or mismatched"} + return True, {"decision_id": decision_id, "status": "delivered"} + + def get_online_devices() -> Tuple[bool, Dict[str, Any]]: """获取在线设备列表 diff --git a/unilabos/app/web/event_bus.py b/unilabos/app/web/event_bus.py new file mode 100644 index 000000000..2fe88d49b --- /dev/null +++ b/unilabos/app/web/event_bus.py @@ -0,0 +1,115 @@ +"""Host 微后端实时事件总线。 + +接口形状与 ``feat/edge-networking-and-scheduler`` 的 MonitorBus 保持一致： +生产端非阻塞写入环形历史，前端通过 SSE 接收增量，并用 REST snapshot 自愈。 +""" + +from __future__ import annotations + +import json +import queue +import threading +import time +from collections import deque +from typing import Any, Deque, Dict, List, Optional, Set, Tuple + + +CHANNELS = ("action",) + + +class MonitorBus: + """线程安全的进程内事件总线和有界历史缓冲。""" + + def __init__(self, history: int = 400, subscriber_buffer: int = 500): + self._lock = threading.Lock() + self._history: Deque[Dict[str, Any]] = deque(maxlen=history) + self._subs: Dict[ + int, + Tuple["queue.Queue[Dict[str, Any]]", Optional[Set[str]]], + ] = {} + self._seq = 0 + self._next_sub_id = 0 + self._subscriber_buffer = subscriber_buffer + + def emit( + self, + channel: str, + event_type: str, + data: Optional[Dict[str, Any]] = None, + ) -> None: + """发布事件；观测链路失败不得阻断设备与调度执行。""" + + try: + with self._lock: + self._seq += 1 + event = { + "seq": self._seq, + "ts": time.time(), + "channel": channel, + "type": event_type, + "data": data or {}, + "trace_id": "", + "span_id": "", + } + self._history.append(event) + for subscriber_queue, channels in self._subs.values(): + if channels is not None and channel not in channels: + continue + try: + subscriber_queue.put_nowait(event) + except queue.Full: + # 前端根据 seq 空洞重新拉 snapshot，不允许慢消费者反压执行。 + pass + except Exception: # noqa: BLE001 - 观测故障必须 fail-open + pass + + def subscribe( + self, + channels: Optional[Set[str]] = None, + backlog: int = 0, + ) -> Tuple[int, "queue.Queue[Dict[str, Any]]", List[Dict[str, Any]]]: + """注册订阅者，返回订阅 ID、增量队列和历史回放。""" + + with self._lock: + self._next_sub_id += 1 + sub_id = self._next_sub_id + subscriber_queue: "queue.Queue[Dict[str, Any]]" = queue.Queue( + maxsize=self._subscriber_buffer + ) + self._subs[sub_id] = (subscriber_queue, channels) + replay = [ + event + for event in self._history + if channels is None or event["channel"] in channels + ] + if backlog <= 0: + replay = [] + else: + replay = replay[-backlog:] + return sub_id, subscriber_queue, replay + + def unsubscribe(self, sub_id: int) -> None: + with self._lock: + self._subs.pop(sub_id, None) + + def recent(self, channel: str, limit: int = 40) -> List[Dict[str, Any]]: + with self._lock: + events = [event for event in self._history if event["channel"] == channel] + return events[-limit:] + + +def format_sse_event(event: Dict[str, Any]) -> str: + """按 Edge monitor 契约编码一条 SSE 事件。""" + + payload = json.dumps(event, ensure_ascii=False, default=str) + return ( + f"id: {event['seq']}\n" + f"event: {event['channel']}\n" + f"data: {payload}\n\n" + ) + + +monitor_bus = MonitorBus() + + +__all__ = ["CHANNELS", "MonitorBus", "format_sse_event", "monitor_bus"] diff --git a/unilabos/app/ws_client.py b/unilabos/app/ws_client.py index 3aa08d707..ce1170de1 100644 --- a/unilabos/app/ws_client.py +++ b/unilabos/app/ws_client.py @@ -27,6 +27,7 @@ from typing_extensions import TypedDict from unilabos.app.model import JobAddReq +from unilabos.registry.action_policy import ERROR_DECISION_TARGET_BACKEND from unilabos.resources.resource_tracker import ResourceDictType from unilabos.ros.nodes.presets.host_node import HostNode from unilabos.utils.type_check import serialize_result_info @@ -68,6 +69,7 @@ class QueueItem: device_action_key: str next_run_time: float = 0 # 下次执行时间戳 retry_count: int = 0 # 重试次数 + error_decision_target: str = ERROR_DECISION_TARGET_BACKEND @dataclass @@ -661,6 +663,8 @@ async def _process_message(self, message_type: str, message_data: Dict[str, Any] await self._handle_device_manage(message_data, "remove") elif message_type == "request_restart": await self._handle_request_restart(message_data) + elif message_type == "job_error_decision": + await self._handle_job_error_decision(message_data) else: logger.debug(f"[MessageProcessor] Unknown message type: {message_type}") @@ -753,6 +757,34 @@ async def _handle_query_action_lock(self, data: Dict[str, Any]): self.websocket_client.report_all_action_locks() logger.trace("[MessageProcessor] query_action_lock: re-reported all action locks") + async def _handle_job_error_decision(self, data: Dict[str, Any]): + """Route one approved error option/result to the exact pending job.""" + + decision_id = str(data.get("decision_id") or "") + job_id = str(data.get("job_id") or "") + device_id = str(data.get("device_id") or "") + if not decision_id and not job_id: + logger.warning("[MessageProcessor] job_error_decision missing decision_id and job_id") + return + if not device_id: + logger.warning("[MessageProcessor] job_error_decision missing device_id") + return + + host_node = HostNode.get_instance(0) + if host_node is None: + logger.warning(f"[MessageProcessor] HostNode unavailable, drop error decision job={job_id[:8]}") + return + if not host_node.handle_action_error_decision( + decision_id, + job_id, + dict(data), + decision_target=ERROR_DECISION_TARGET_BACKEND, + ): + logger.warning( + f"[MessageProcessor] No pending error decision matched " + f"decision={decision_id} job={job_id[:8]} device={device_id}" + ) + async def _handle_job_start(self, data: Dict[str, Any]): """处理job_start消息：服务端直接下发，本地直跑或排队(不再要求先 query_action_state)。""" try: @@ -1683,6 +1715,25 @@ def publish_job_status( logger.trace(f"[WebSocketClient] Job status published: {job_log} - {status}") + def publish_job_error_decision_required(self, report: Dict[str, Any]) -> bool: + """Send an action exception and its class-matched options for approval.""" + + if self.is_disabled or not self.is_connected(): + logger.warning( + f"[WebSocketClient] Cannot report action error while disconnected: " + f"job={str(report.get('job_id', ''))[:8]} " + f"exception={report.get('exception_type', '')}" + ) + return False + message = {"action": "job_error_decision_required", "data": report} + queued = self.message_processor.send_message(message) + if queued: + logger.info( + f"[WebSocketClient] Action error awaiting decision: " + f"decision={report.get('decision_id')} job={str(report.get('job_id', ''))[:8]}" + ) + return queued + def send_ping(self, ping_id: str, timestamp: float) -> None: """发送ping消息""" if self.is_disabled or not self.is_connected(): diff --git a/unilabos/registry/action_policy.py b/unilabos/registry/action_policy.py new file mode 100644 index 000000000..b7e9eefe2 --- /dev/null +++ b/unilabos/registry/action_policy.py @@ -0,0 +1,173 @@ +"""Action exception policies shared by registry and runtime code.""" + +from __future__ import annotations + +from copy import deepcopy +from typing import Any, Dict, List, Literal, Mapping, NotRequired, TypedDict + + +DEFAULT_ERROR_CLASS = "*" + +SUCCESS_TYPE_NORMAL = "normal" +SUCCESS_TYPE_SKIP = "skip" +SUCCESS_TYPE_OPERATOR_INTERVENTION = "operator_intervention" +SuccessType = Literal["normal", "skip", "operator_intervention"] + +ERROR_DECISION_TARGET_BACKEND = "backend" +ERROR_DECISION_TARGET_MICRO_BACKEND = "micro_backend" + + +class FallbackAction(TypedDict): + """Server-side single action executed after operator approval.""" + + action_name: str + params: NotRequired[Dict[str, Any]] + + +class ErrorPolicyOption(TypedDict): + """One option displayed for a matched exception class.""" + + action: str + label: str + description: NotRequired[str] + fallback_action: NotRequired[FallbackAction] + + +class ErrorPolicy(TypedDict): + """Exception class name -> approval options for one ``@action``.""" + + options: Dict[str, List[ErrorPolicyOption]] + max_retries: NotRequired[int] + decision_timeout_seconds: NotRequired[float] + default_on_decision_timeout: NotRequired[Literal["abort", "retry", "skip"]] + + +def _normalize_fallback_action(value: Any) -> FallbackAction: + if isinstance(value, str): + if not value: + raise ValueError("fallback_action action_name 不能为空") + return {"action_name": value, "params": {}} + if not isinstance(value, Mapping): + raise TypeError("fallback_action 必须是动作名字符串或字典") + + action_name = value.get("action_name") or value.get("name") + if not isinstance(action_name, str) or not action_name: + raise ValueError("fallback_action.action_name 必须是非空字符串") + params = value.get("params", {}) + if not isinstance(params, Mapping): + raise TypeError("fallback_action.params 必须是字典") + return {"action_name": action_name, "params": deepcopy(dict(params))} + + +def _normalize_option(value: Any) -> ErrorPolicyOption: + if not isinstance(value, Mapping): + raise TypeError("error_policy option 必须是字典") + action = value.get("action") + label = value.get("label") + if not isinstance(action, str) or not action.strip(): + raise ValueError("error_policy option.action 必须是非空字符串") + if not isinstance(label, str) or not label.strip(): + raise ValueError("error_policy option.label 必须是非空字符串") + + option: ErrorPolicyOption = { + "action": action.strip(), + "label": label.strip(), + } + description = value.get("description") + if description is not None: + option["description"] = str(description) + if value.get("fallback_action") is not None: + option["fallback_action"] = _normalize_fallback_action( + value["fallback_action"] + ) + return option + + +def normalize_error_policy( + policy: Mapping[str, Any] | None, +) -> Dict[str, Any] | None: + """Validate and copy a policy into a registry-safe representation. + + ``options`` is keyed by exception class name. A legacy flat option list is + accepted as the ``"*"`` fallback to ease selective migration. + """ + + if not policy: + return None + raw_options = policy.get("options") + if isinstance(raw_options, list): + raw_options = {DEFAULT_ERROR_CLASS: raw_options} + if not isinstance(raw_options, Mapping) or not raw_options: + raise ValueError("error_policy.options 必须是非空的异常类名到 option 列表映射") + + options: Dict[str, List[ErrorPolicyOption]] = {} + for error_class_name, raw_class_options in raw_options.items(): + if not isinstance(error_class_name, str) or not error_class_name: + raise ValueError("error_policy.options 的异常类名必须是非空字符串") + if not isinstance(raw_class_options, list) or not raw_class_options: + raise ValueError( + f"error_policy.options[{error_class_name!r}] 必须是非空列表" + ) + normalized_options = [ + _normalize_option(option) for option in raw_class_options + ] + actions = [option["action"] for option in normalized_options] + if len(actions) != len(set(actions)): + raise ValueError( + f"error_policy.options[{error_class_name!r}] 包含重复 action" + ) + options[error_class_name] = normalized_options + + normalized: Dict[str, Any] = {"options": options} + max_retries = policy.get("max_retries", 3) + if isinstance(max_retries, bool) or not isinstance(max_retries, int) or max_retries < 0: + raise ValueError("error_policy.max_retries 必须是非负整数") + normalized["max_retries"] = max_retries + + decision_timeout = policy.get("decision_timeout_seconds", 300.0) + if ( + isinstance(decision_timeout, bool) + or not isinstance(decision_timeout, (int, float)) + or decision_timeout <= 0 + ): + raise ValueError("error_policy.decision_timeout_seconds 必须大于 0") + normalized["decision_timeout_seconds"] = float(decision_timeout) + + timeout_action = policy.get("default_on_decision_timeout", "abort") + if timeout_action not in {"abort", "retry", "skip"}: + raise ValueError("default_on_decision_timeout 仅支持 abort/retry/skip") + normalized["default_on_decision_timeout"] = timeout_action + return normalized + + +def resolve_error_options( + policy: Mapping[str, Any] | None, + exc: BaseException, +) -> List[Dict[str, Any]]: + """Resolve options by exception MRO, then the ``*`` fallback.""" + + return resolve_error_options_by_names( + policy, + [error_class.__name__ for error_class in type(exc).__mro__], + ) + + +def resolve_error_options_by_names( + policy: Mapping[str, Any] | None, + error_class_names: List[str], +) -> List[Dict[str, Any]]: + """Host 根据设备回传的异常 MRO 名称解析注册表策略。""" + + if not isinstance(policy, Mapping): + return [] + options = policy.get("options") + if not isinstance(options, Mapping): + return [] + for error_class_name in error_class_names: + if not isinstance(error_class_name, str): + continue + matched = options.get(error_class_name) + if isinstance(matched, list): + return deepcopy(matched) + fallback = options.get(DEFAULT_ERROR_CLASS) + return deepcopy(fallback) if isinstance(fallback, list) else [] diff --git a/unilabos/registry/ast_registry_scanner.py b/unilabos/registry/ast_registry_scanner.py index 9b29fef2c..c7775daaa 100644 --- a/unilabos/registry/ast_registry_scanner.py +++ b/unilabos/registry/ast_registry_scanner.py @@ -36,7 +36,7 @@ MAX_SCAN_DEPTH = 10 # 最大目录递归深度 MAX_SCAN_FILES = 1000 # 最大扫描文件数量 -_CACHE_VERSION = 6 # 缓存格式版本号，格式变更时递增 +_CACHE_VERSION = 7 # 缓存格式版本号，格式变更时递增 _DEVICE_ID_RE = re.compile(r"^[A-Za-z0-9_]+$") # 合法的装饰器来源模块 @@ -896,6 +896,11 @@ def _extract_class_body( action_args.setdefault("description", "") action_args.setdefault("auto_prefix", False) action_args.setdefault("parent", False) + action_args.setdefault("error_policy", None) + if action_args["error_policy"]: + from unilabos.registry.action_policy import normalize_error_policy + + action_args["error_policy"] = normalize_error_policy(action_args["error_policy"]) method_params = _extract_method_params(item, import_map) return_type = _get_annotation_str(item.returns, import_map) is_async = isinstance(item, ast.AsyncFunctionDef) diff --git a/unilabos/registry/decorators.py b/unilabos/registry/decorators.py index b31d732f4..9dc60fd37 100644 --- a/unilabos/registry/decorators.py +++ b/unilabos/registry/decorators.py @@ -357,6 +357,7 @@ def action( parent: bool = False, node_type: Optional["NodeType"] = None, feedback_interval: Optional[float] = None, + error_policy: Optional[Dict[str, Any]] = None, ): """ 动作方法装饰器 @@ -389,6 +390,8 @@ def AddProtocol(self): ... parent: 若为 True，当方法参数为空 (*args, **kwargs) 时，通过 MRO 从父类获取真实方法参数 node_type: 动作的节点类型 (NodeType.ILAB / NodeType.MANUAL_CONFIRM)。 不填写时不写入注册表。 + error_policy: 按异常类名匹配审批选项的策略。结构见 + unilabos.registry.action_policy.ErrorPolicy。 """ def decorator(func: F) -> F: @@ -424,7 +427,14 @@ def wrapper(*args, **kwargs): meta["feedback_interval"] = feedback_interval if node_type is not None: meta["node_type"] = node_type.value if isinstance(node_type, NodeType) else str(node_type) + normalized_error_policy = None + if error_policy: + from unilabos.registry.action_policy import normalize_error_policy + + normalized_error_policy = normalize_error_policy(error_policy) + meta["error_policy"] = normalized_error_policy wrapper._action_registry_meta = meta # type: ignore[attr-defined] + wrapper._action_error_policy = normalized_error_policy # type: ignore[attr-defined] # 设置 _is_always_free 保持与旧 @always_free 装饰器兼容 if always_free: diff --git a/unilabos/registry/registry.py b/unilabos/registry/registry.py index 590ff45e2..d41a539de 100644 --- a/unilabos/registry/registry.py +++ b/unilabos/registry/registry.py @@ -977,6 +977,8 @@ def _build_json_command_entry(method_name, method_info, action_args=None): entry["always_free"] = True _fb_iv = (action_args or {}).get("feedback_interval", method_info.get("feedback_interval", 1.0)) entry["feedback_interval"] = _fb_iv + if (action_args or {}).get("error_policy"): + entry["error_policy"] = action_args["error_policy"] nt = normalize_enum_value((action_args or {}).get("node_type"), NodeType) if nt: entry["node_type"] = nt @@ -1115,6 +1117,8 @@ def _build_json_command_entry(method_name, method_info, action_args=None): action_entry["always_free"] = True _fb_iv = action_args.get("feedback_interval", method_info.get("feedback_interval", 1.0)) action_entry["feedback_interval"] = _fb_iv + if action_args.get("error_policy"): + action_entry["error_policy"] = action_args["error_policy"] nt = normalize_enum_value(action_args.get("node_type"), NodeType) if nt: action_entry["node_type"] = nt diff --git a/unilabos/ros/nodes/base_device_node.py b/unilabos/ros/nodes/base_device_node.py index 23e905d21..edaf5ebdc 100644 --- a/unilabos/ros/nodes/base_device_node.py +++ b/unilabos/ros/nodes/base_device_node.py @@ -37,6 +37,9 @@ from unilabos.config.config import BasicConfig from unilabos.registry.decorators import get_topic_config +from unilabos.registry.action_policy import ( + SUCCESS_TYPE_NORMAL, +) from unilabos.registry.placeholder_type import ResourceSlotRawInput from unilabos.utils.decorator import get_all_subscriptions @@ -460,7 +463,6 @@ def __init__( self._cross_device_action_clients: Dict[str, ActionClient] = {} # 跨设备动作类型探测缓存（key: "/"，value: 原生 Action 类型或 None） self._remote_action_type_cache: Dict[str, Any] = {} - # 创建线程池执行器 self._executor = ThreadPoolExecutor( max_workers=max(len(action_value_mappings), 1), thread_name_prefix=f"ROSDevice{self.device_id}" @@ -2003,6 +2005,22 @@ def get_real_function(self, instance, attr_name): obj = getattr(instance, attr_name) return obj, get_type_hints(obj) + def _resolve_report_action_name( + self, + action_name: str, + action_kwargs: Dict[str, Any], + ) -> str: + """解析 JSON command 实际调用的业务动作名，供 Host 匹配注册表。""" + + report_action_name = action_name + if action_name in {"_execute_driver_command", "_execute_driver_command_async"}: + try: + command = json.loads(action_kwargs.get("string", "")) + report_action_name = str(command["function_name"]) + except (KeyError, TypeError, ValueError, json.JSONDecodeError): + pass + return report_action_name + def _create_execute_callback(self, action_name, action_value_mapping): """创建动作执行回调函数""" @@ -2011,6 +2029,8 @@ async def execute_callback(goal_handle: ServerGoalHandle): execution_error = "" execution_success = False action_return_value = None + execution_suc_type = SUCCESS_TYPE_NORMAL + execution_error_info = None ##### self.lab_logger().info(f"执行动作: {action_name}") goal = goal_handle.request @@ -2036,6 +2056,11 @@ def ACTION(**kwargs): action_kwargs = convert_from_ros_msg_with_mapping(goal, action_value_mapping["goal"]) self.lab_logger().debug(f"任务 {ACTION.__name__} 接收到原始目标: {str(action_kwargs)[:1000]}") self.lab_logger().trace(f"任务 {ACTION.__name__} 接收到原始目标: {action_kwargs}") + report_action_name = self._resolve_report_action_name( + action_name, + action_kwargs, + ) + error_skip = False # 向Host查询物料当前状态，如果是host本身的增加物料的请求，则直接跳过 if action_name not in ["create_resource_detailed", "create_resource"]: @@ -2231,6 +2256,28 @@ async def _wake(): execution_success = True action_return_value = _raw_result + if isinstance(_raw_result, BaseException): + execution_error_info = { + "action_name": report_action_name, + "exception_type": type(_raw_result).__name__, + "exception_mro": [ + error_class.__name__ + for error_class in type(_raw_result).__mro__ + ], + "error_message": str(_raw_result), + "traceback": execution_error, + } + category = getattr(_raw_result, "category", None) + severity = getattr(_raw_result, "severity", None) + if category is not None: + execution_error_info["category"] = str( + getattr(category, "value", category) + ) + if severity is not None: + execution_error_info["severity"] = str( + getattr(severity, "value", severity) + ) + # 清理 feedback timer if _feedback_timer is not None: _feedback_timer.cancel() @@ -2308,7 +2355,13 @@ async def _wake(): setattr( result_msg, attr_name, - get_result_info_str(execution_error, execution_success, action_return_value), + get_result_info_str( + execution_error, + execution_success, + action_return_value, + suc_type=execution_suc_type, + error_info=execution_error_info, + ), ) self.lab_logger().trace(f"动作 {action_name} 完成并返回结果") diff --git a/unilabos/ros/nodes/presets/host_node.py b/unilabos/ros/nodes/presets/host_node.py index ffa5698af..8afffebee 100644 --- a/unilabos/ros/nodes/presets/host_node.py +++ b/unilabos/ros/nodes/presets/host_node.py @@ -4,10 +4,11 @@ import time import traceback import uuid +from copy import deepcopy from unilabos.utils.tools import fast_dumps_str as _fast_dumps_str, fast_loads as _fast_loads from dataclasses import dataclass, field -from typing import TYPE_CHECKING, Optional, Dict, Any, List, ClassVar, Set, Tuple, Union +from typing import TYPE_CHECKING, Optional, Dict, Any, List, ClassVar, Mapping, Set, Tuple, Union from action_msgs.msg import GoalStatus from geometry_msgs.msg import Point @@ -27,6 +28,13 @@ from unique_identifier_msgs.msg import UUID from unilabos.registry.decorators import device, action, NodeType, ActionInputHandle, ActionOutputHandle, DataSource +from unilabos.registry.action_policy import ( + ERROR_DECISION_TARGET_BACKEND, + ERROR_DECISION_TARGET_MICRO_BACKEND, + SUCCESS_TYPE_OPERATOR_INTERVENTION, + SUCCESS_TYPE_SKIP, + resolve_error_options_by_names, +) from unilabos.registry.placeholder_type import ( ResourceSlot, DeviceSlot, @@ -360,6 +368,10 @@ def __init__( } # device_id -> action_value_mappings(本地+远程设备统一存储) self._slave_registry_configs: Dict[str, Dict] = {} # registry_name -> registry_config(含action_value_mappings) self._goals: Dict[str, Any] = {} # 用来存储多个目标的状态 + # 异常决策只存在于 Host：设备返回结构化失败，Host 保留原 job 并负责恢复动作。 + self._error_execution_contexts: Dict[str, Dict[str, Any]] = {} + self._pending_action_error_decisions: Dict[str, Dict[str, Any]] = {} + self._pending_action_error_decisions_lock = threading.RLock() self._online_devices: Set[str] = {f"{self.namespace}/{device_id}"} # 用于跟踪在线设备 self._last_discovery_time = 0.0 # 上次设备发现的时间 self._discovery_lock = threading.Lock() # 设备发现的互斥锁 @@ -857,6 +869,11 @@ def send_goal( action_kwargs: Dict[str, Any], sample_material: Dict[str, str], server_info: Optional[Dict[str, Any]] = None, + *, + transport_goal_id: Optional[str] = None, + result_item: Optional["QueueItem"] = None, + recovery_suc_type: Optional[str] = None, + cache_error_context: bool = True, ) -> None: """ 向设备发送目标请求 @@ -866,9 +883,11 @@ def send_goal( action_kwargs: 动作参数 server_info: 服务器发送信息，包含发送时间戳等 """ - u = uuid.UUID(item.job_id) + callback_item = result_item or item + u = uuid.UUID(transport_goal_id or item.job_id) device_id = item.device_id action_name = item.action_name + original_action_kwargs = dict(action_kwargs) if BasicConfig.test_mode: action_id = f"/devices/{device_id}/{action_name}" @@ -901,21 +920,50 @@ def send_goal( if action_id not in self._action_clients: raise ValueError(f"ActionClient {action_id} not found.") - action_client: ActionClient = self._action_clients[action_id] - goal_msg = convert_to_ros_msg(action_client._action_type.Goal(), action_kwargs) - - # self.lab_logger().trace(f"[Host Node] Sending goal for {action_id}: {str(goal_msg)[:1000]}") - self.lab_logger().trace(f"[Host Node] Sending goal for {action_id}: {action_kwargs}") - self.lab_logger().trace(f"[Host Node] Sending goal for {action_id}: {goal_msg}") - action_client.wait_for_server() - goal_uuid_obj = UUID(uuid=list(u.bytes)) + context_cached = False + if cache_error_context: + existing = self._error_execution_contexts.get(callback_item.job_id, {}) + self._error_execution_contexts[callback_item.job_id] = { + "item": callback_item, + "action_type": action_type, + "action_kwargs": original_action_kwargs, + "sample_material": dict(sample_material), + "server_info": dict(server_info) if server_info else None, + "retry_count": int(existing.get("retry_count", 0)), + } + context_cached = True - future = action_client.send_goal_async( - goal_msg, - feedback_callback=lambda feedback_msg: self.feedback_callback(item, action_id, feedback_msg), - goal_uuid=goal_uuid_obj, + try: + action_client: ActionClient = self._action_clients[action_id] + goal_msg = convert_to_ros_msg(action_client._action_type.Goal(), action_kwargs) + + # self.lab_logger().trace(f"[Host Node] Sending goal for {action_id}: {str(goal_msg)[:1000]}") + self.lab_logger().trace(f"[Host Node] Sending goal for {action_id}: {action_kwargs}") + self.lab_logger().trace(f"[Host Node] Sending goal for {action_id}: {goal_msg}") + action_client.wait_for_server() + goal_uuid_obj = UUID(uuid=list(u.bytes)) + + future = action_client.send_goal_async( + goal_msg, + feedback_callback=lambda feedback_msg: self.feedback_callback( + callback_item, + action_id, + feedback_msg, + ), + goal_uuid=goal_uuid_obj, + ) + except Exception: + if context_cached: + self._error_execution_contexts.pop(callback_item.job_id, None) + raise + future.add_done_callback( + lambda f: self.goal_response_callback( + callback_item, + action_id, + f, + recovery_suc_type=recovery_suc_type, + ) ) - future.add_done_callback(lambda f: self.goal_response_callback(item, action_id, f)) def _build_test_mode_return( self, device_id: str, action_name: str, action_kwargs: Dict[str, Any] @@ -961,18 +1009,563 @@ def _handle_test_mode_result( for bridge in self.bridges: if hasattr(bridge, "publish_job_status"): bridge.publish_job_status(mock_return, item, status, return_info) + self._emit_local_action_event( + item, + "job_status", + self._job_status_event_data( + item, + status, + mock_return, + return_info, + ), + ) + self._error_execution_contexts.pop(job_id, None) + + @staticmethod + def _job_status_event_data( + item: "QueueItem", + status: str, + feedback_data: Dict[str, Any], + return_info: Optional[Dict[str, Any]] = None, + ) -> Dict[str, Any]: + """生成与边云 ``job_status.data`` 一致的本地事件载荷。""" + + return { + "job_id": item.job_id, + "task_id": item.task_id, + "device_id": item.device_id, + "notebook_id": item.notebook_id, + "action_name": item.action_name, + "status": status, + "feedback_data": feedback_data, + "return_info": return_info, + "timestamp": time.time(), + } + + @staticmethod + def _emit_local_action_event( + item: "QueueItem", + event_type: str, + data: Dict[str, Any], + ) -> None: + """向 Host 微后端发布事件；事件系统故障不得影响 action。""" + + if ( + getattr(item, "error_decision_target", ERROR_DECISION_TARGET_BACKEND) + != ERROR_DECISION_TARGET_MICRO_BACKEND + ): + return + try: + from unilabos.app.web.event_bus import monitor_bus + + monitor_bus.emit("action", event_type, data) + except Exception: # noqa: BLE001 - 观测链路必须 fail-open + pass + + def _finish_error_handled_job( + self, + item: "QueueItem", + status: str, + return_info: Dict[str, Any], + result_data: Dict[str, Any], + ) -> None: + """结束由 Host 管理的异常决策 job，并释放正常队列状态。""" + + job_id = item.job_id + self._goals.pop(job_id, None) + self._error_execution_contexts.pop(job_id, None) + with self._pending_action_error_decisions_lock: + stale_ids = [ + decision_id + for decision_id, pending in self._pending_action_error_decisions.items() + if pending.get("job_id") == job_id + ] + for decision_id in stale_ids: + pending = self._pending_action_error_decisions.pop(decision_id) + timer = pending.get("timer") + if timer is not None: + timer.cancel() + + try: + from unilabos.app.web.controller import store_job_result + + store_job_result(job_id, status, return_info, result_data) + except ImportError: + pass + except Exception as ex: # noqa: BLE001 - 不能阻断队列终态上报 + self.lab_logger().warning( + f"[Host Node] Store job result failed for {job_id[:8]}: {ex}" + ) + + for bridge in self.bridges: + if hasattr(bridge, "publish_job_status"): + bridge.publish_job_status(result_data, item, status, return_info) + self._emit_local_action_event( + item, + "job_status", + self._job_status_event_data( + item, + status, + result_data, + return_info, + ), + ) + + def _begin_action_error_decision( + self, + item: "QueueItem", + return_info: Dict[str, Any], + result_data: Dict[str, Any], + ) -> bool: + """设备失败后在 Host 创建决策点；成功上报时原 job 继续保持 pending。""" + + raw_error_info = return_info.get("error_info") + if not isinstance(raw_error_info, dict): + return False + action_mappings = self._action_value_mappings.get(item.device_id, {}) + report_action_name = str( + raw_error_info.get("action_name") or item.action_name + ) + candidates = [report_action_name, item.action_name] + candidates.extend( + f"auto-{candidate}" + for candidate in list(candidates) + if not candidate.startswith("auto-") + ) + policy = None + for candidate in candidates: + mapping = action_mappings.get(candidate) + if isinstance(mapping, dict) and mapping.get("error_policy"): + policy = mapping["error_policy"] + break + if not isinstance(policy, Mapping): + return False + exception_mro = raw_error_info.get("exception_mro") + if not isinstance(exception_mro, list): + exception_mro = [ + str(raw_error_info.get("exception_type") or "Exception") + ] + options = resolve_error_options_by_names(policy, exception_mro) + if not options: + return False + error_info = { + **raw_error_info, + "options": options, + "max_retries": int(policy.get("max_retries", 3)), + "decision_timeout_seconds": float( + policy.get("decision_timeout_seconds", 300.0) + ), + "default_on_decision_timeout": str( + policy.get("default_on_decision_timeout", "abort") + ), + } + decision_target = str( + getattr(item, "error_decision_target", ERROR_DECISION_TARGET_BACKEND) + ) + if decision_target not in { + ERROR_DECISION_TARGET_BACKEND, + ERROR_DECISION_TARGET_MICRO_BACKEND, + }: + self.lab_logger().warning( + f"[Host Node] 未知异常决策目标 {decision_target!r}，按后端通道处理" + ) + decision_target = ERROR_DECISION_TARGET_BACKEND + + execution_context = self._error_execution_contexts.get(item.job_id) + if execution_context is None: + self.lab_logger().warning( + f"[Host Node] Job {item.job_id[:8]} 缺少重试上下文，仅支持 skip/abort" + ) + + decision_id = str(uuid.uuid4()) + pending = { + "decision_id": decision_id, + "job_id": item.job_id, + "item": item, + "return_info": dict(return_info), + "result_data": dict(result_data), + "error_info": dict(error_info), + "execution_context": execution_context, + "decision_target": decision_target, + "report": None, + "resolving": False, + "timer": None, + } + with self._pending_action_error_decisions_lock: + self._pending_action_error_decisions[decision_id] = pending + + created_at = time.time() + timeout_seconds = float(error_info.get("decision_timeout_seconds", 300.0)) + report = { + "decision_id": decision_id, + "device_id": item.device_id, + "action_name": error_info.get("action_name") or item.action_name, + "task_id": item.task_id, + "job_id": item.job_id, + "exception_type": error_info.get("exception_type", "Exception"), + "error_message": error_info.get("error_message", return_info.get("error", "")), + "traceback": error_info.get("traceback", return_info.get("error", "")), + "options": options, + "retry_count": int((execution_context or {}).get("retry_count", 0)), + "max_retries": int(error_info.get("max_retries", 3)), + "created_at": created_at, + "decision_timeout_seconds": timeout_seconds, + "expires_at": created_at + timeout_seconds, + "default_on_decision_timeout": error_info.get( + "default_on_decision_timeout", + "abort", + ), + "require_confirmation": True, + } + for key in ("category", "severity"): + if error_info.get(key) is not None: + report[key] = error_info[key] + pending["report"] = report + + self._goals.pop(item.job_id, None) + timer = threading.Timer( + timeout_seconds, + self._handle_action_error_decision_timeout, + args=(decision_id,), + ) + timer.daemon = True + pending["timer"] = timer + timer.start() + + # 本地任务由微后端直接读取 Host pending；云端任务只投递给后端 bridge。 + accepted = decision_target == ERROR_DECISION_TARGET_MICRO_BACKEND + if decision_target == ERROR_DECISION_TARGET_BACKEND: + for bridge in self.bridges: + publish = getattr(bridge, "publish_job_error_decision_required", None) + if not callable(publish): + continue + try: + if publish(report): + accepted = True + break + except Exception as ex: # noqa: BLE001 - 逐个尝试决策通道 + self.lab_logger().warning( + f"[Host Node] 异常决策通道失败: {bridge!r}: {ex}" + ) + + if not accepted: + with self._pending_action_error_decisions_lock: + removed = self._pending_action_error_decisions.pop( + decision_id, + None, + ) + if removed is not None: + timer.cancel() + return False + + with self._pending_action_error_decisions_lock: + still_pending = decision_id in self._pending_action_error_decisions + if still_pending: + self._emit_local_action_event( + item, + "job_error_decision_required", + deepcopy(report), + ) + self.lab_logger().info( + f"[Host Node] Job {item.job_id[:8]} 等待异常决策 " + f"{decision_id} target={decision_target}" + ) + return True + + def get_pending_action_error_decisions( + self, + decision_target: Optional[str] = None, + ) -> List[Dict[str, Any]]: + """按决策目标查询 Host 持有的 pending 报告。""" + + with self._pending_action_error_decisions_lock: + reports = [ + deepcopy(pending["report"]) + for pending in self._pending_action_error_decisions.values() + if pending.get("report") is not None + and ( + decision_target is None + or pending.get("decision_target") == decision_target + ) + ] + return reports + + def _handle_action_error_decision_timeout(self, decision_id: str) -> None: + with self._pending_action_error_decisions_lock: + pending = self._pending_action_error_decisions.get(decision_id) + if pending is None: + return + error_info = pending["error_info"] + job_id = pending["job_id"] + self.handle_action_error_decision( + decision_id, + job_id, + { + "decision_id": decision_id, + "job_id": job_id, + "action": error_info.get("default_on_decision_timeout", "abort"), + "reason": "decision_timeout", + }, + ) + + def handle_action_error_decision( + self, + decision_id: str, + job_id: str, + decision: Dict[str, Any], + *, + decision_target: Optional[str] = None, + ) -> bool: + """在 Host 上处理决策，并通过现有 ActionClient 发起恢复动作。""" + + with self._pending_action_error_decisions_lock: + pending = self._pending_action_error_decisions.get(decision_id) if decision_id else None + if pending is None and job_id: + matches = [ + candidate + for candidate in self._pending_action_error_decisions.values() + if candidate.get("job_id") == job_id + ] + pending = matches[0] if len(matches) == 1 else None + if pending is None or pending.get("resolving"): + return False + if ( + decision_target is not None + and pending.get("decision_target") != decision_target + ): + return False + if job_id and pending["job_id"] != job_id: + return False + body_decision_id = str(decision.get("decision_id") or "") + body_job_id = str(decision.get("job_id") or "") + if body_decision_id and body_decision_id != pending["decision_id"]: + return False + if body_job_id and body_job_id != pending["job_id"]: + return False + body_device_id = str(decision.get("device_id") or "") + if body_device_id and body_device_id != pending["item"].device_id: + return False + + selected_option = decision.get("option") + if isinstance(selected_option, dict): + selected = str(selected_option.get("action") or "abort") + for result_key in ("result", "return_value"): + if result_key not in decision and result_key in selected_option: + decision[result_key] = selected_option[result_key] + else: + selected = str(decision.get("action") or selected_option or "abort") + options = pending["error_info"]["options"] + option = next( + (candidate for candidate in options if str(candidate.get("action")) == selected), + None, + ) + if option is None and decision.get("reason") != "decision_timeout": + return False + + pending["resolving"] = True + self._pending_action_error_decisions.pop(pending["decision_id"], None) + timer = pending.get("timer") + if timer is not None: + timer.cancel() + + item = pending["item"] + self._emit_local_action_event( + item, + "job_error_decision_resolved", + { + "decision_id": pending["decision_id"], + "job_id": pending["job_id"], + "task_id": item.task_id, + "device_id": item.device_id, + "action_name": item.action_name, + "selected_action": selected, + "reason": str(decision.get("reason") or ""), + "resolved_at": time.time(), + }, + ) + if selected == "abort": + self._finish_error_handled_job( + item, + "failed", + pending["return_info"], + pending["result_data"], + ) + return True + + if selected == "skip": + return_value = decision.get("result", decision.get("return_value")) + return_info = serialize_result_info( + "", + True, + return_value, + suc_type=SUCCESS_TYPE_SKIP, + ) + result_data = dict(pending["result_data"]) + result_data["return_info"] = json.dumps(return_info, ensure_ascii=False) + self._finish_error_handled_job(item, "success", return_info, result_data) + return True + + execution_context = pending.get("execution_context") + if selected == "retry": + if execution_context is None: + self._finish_error_handled_job( + item, + "failed", + serialize_result_info("缺少原动作上下文，无法重试", False, {}), + pending["result_data"], + ) + return True + retries = int(execution_context.get("retry_count", 0)) + max_retries = int(pending["error_info"].get("max_retries", 3)) + if retries >= max_retries: + self._finish_error_handled_job( + item, + "failed", + serialize_result_info( + f"action {item.action_name} exceeded {max_retries} retries", + False, + {}, + ), + pending["result_data"], + ) + return True + execution_context["retry_count"] = retries + 1 + try: + self.send_goal( + execution_context["item"], + execution_context["action_type"], + execution_context["action_kwargs"], + execution_context["sample_material"], + execution_context["server_info"], + transport_goal_id=str(uuid.uuid4()), + cache_error_context=False, + ) + except Exception as ex: # noqa: BLE001 - 转成原 job 的终态失败 + self._finish_error_handled_job( + item, + "failed", + serialize_result_info(traceback.format_exc(), False, {}), + {"error": str(ex)}, + ) + return True + + fallback = option.get("fallback_action") if isinstance(option, dict) else None + if not isinstance(fallback, dict): + if "result" in decision or "return_value" in decision: + return_value = decision.get("result", decision.get("return_value")) + return_info = serialize_result_info( + "", + True, + return_value, + suc_type=SUCCESS_TYPE_OPERATOR_INTERVENTION, + ) + self._finish_error_handled_job(item, "success", return_info, {}) + return True + self._finish_error_handled_job( + item, + "failed", + serialize_result_info( + f"error option {selected} missing fallback_action", + False, + {}, + ), + pending["result_data"], + ) + return True + + fallback_name = str(fallback.get("action_name") or "") + action_mappings = self._action_value_mappings.get(item.device_id, {}) + fallback_key = fallback_name + mapping = action_mappings.get(fallback_key) + if mapping is None: + fallback_key = f"auto-{fallback_name}" + mapping = action_mappings.get(fallback_key) + if not fallback_name or not isinstance(mapping, dict): + self._finish_error_handled_job( + item, + "failed", + serialize_result_info( + f"fallback action not registered: {fallback_name}", + False, + {}, + ), + pending["result_data"], + ) + return True + + fallback_item = type(item)( + task_type=item.task_type, + device_id=item.device_id, + action_name=fallback_key, + task_id=item.task_id, + job_id=item.job_id, + notebook_id=item.notebook_id, + device_action_key=f"/devices/{item.device_id}/{fallback_key}", + error_decision_target=decision_target or pending["decision_target"], + ) + try: + self.send_goal( + fallback_item, + str(mapping.get("type") or "UniLabJsonCommand"), + dict(fallback.get("params") or {}), + {}, + None, + transport_goal_id=str(uuid.uuid4()), + result_item=item, + recovery_suc_type=SUCCESS_TYPE_OPERATOR_INTERVENTION, + cache_error_context=False, + ) + except Exception as ex: # noqa: BLE001 - 转成原 job 的终态失败 + self._finish_error_handled_job( + item, + "failed", + serialize_result_info(traceback.format_exc(), False, {}), + {"error": str(ex)}, + ) + return True - def goal_response_callback(self, item: "QueueItem", action_id: str, future) -> None: + def goal_response_callback( + self, + item: "QueueItem", + action_id: str, + future, + recovery_suc_type: Optional[str] = None, + ) -> None: """目标响应回调""" - goal_handle = future.result() + try: + goal_handle = future.result() + except Exception as ex: # noqa: BLE001 - 转成 job 终态失败 + self.lab_logger().error( + f"[Host Node] Goal {item.action_name} ({item.job_id}) response failed: {ex}" + ) + self._finish_error_handled_job( + item, + "failed", + serialize_result_info(traceback.format_exc(), False, {}), + {}, + ) + return if not goal_handle.accepted: self.lab_logger().warning(f"[Host Node] Goal {item.action_name} ({item.job_id}) rejected") + self._finish_error_handled_job( + item, + "failed", + serialize_result_info("Goal was rejected", False, {}), + {}, + ) return self.lab_logger().info(f"[Host Node] Goal {action_id} ({item.job_id}) accepted") self._goals[item.job_id] = goal_handle goal_future = goal_handle.get_result_async() - goal_future.add_done_callback(lambda f: self.get_result_callback(item, action_id, f)) + goal_future.add_done_callback( + lambda f: self.get_result_callback( + item, + action_id, + f, + recovery_suc_type=recovery_suc_type, + ) + ) goal_future.result() def feedback_callback(self, item: "QueueItem", action_id: str, feedback_msg) -> None: @@ -984,8 +1577,19 @@ def feedback_callback(self, item: "QueueItem", action_id: str, feedback_msg) -> for bridge in self.bridges: if hasattr(bridge, "publish_job_status"): bridge.publish_job_status(feedback_data, item, "running") + self._emit_local_action_event( + item, + "job_status", + self._job_status_event_data(item, "running", feedback_data), + ) - def get_result_callback(self, item: "QueueItem", action_id: str, future) -> None: + def get_result_callback( + self, + item: "QueueItem", + action_id: str, + future, + recovery_suc_type: Optional[str] = None, + ) -> None: """获取结果回调""" job_id = item.job_id @@ -1035,34 +1639,39 @@ def get_result_callback(self, item: "QueueItem", action_id: str, future) -> None status = "failed" return_info = serialize_result_info("缺少return_info", False, result_data) + if status == "success" and recovery_suc_type: + return_info["suc_type"] = recovery_suc_type + result_data["return_info"] = json.dumps( + return_info, + ensure_ascii=False, + ) + + terminal_result_data = ( + {} if goal_status == GoalStatus.STATUS_CANCELED else result_data + ) + if ( + status == "failed" + and recovery_suc_type is None + and self._begin_action_error_decision( + item, + return_info, + terminal_result_data, + ) + ): + self.lab_logger().info( + f"[Host Node] Result for {action_id} ({job_id[:8]}): awaiting_error_decision" + ) + return + self.lab_logger().info(f"[Host Node] Result for {action_id} ({job_id[:8]}): {status}") if goal_status != GoalStatus.STATUS_CANCELED: self.lab_logger().trace(f"[Host Node] Result data: {result_data}") - - # 清理 _goals 中的记录 - if job_id in self._goals: - del self._goals[job_id] - self.lab_logger().trace(f"[Host Node] Removed goal {job_id[:8]} from _goals") - - # 存储结果供 HTTP API 查询 - try: - from unilabos.app.web.controller import store_job_result - - if goal_status == GoalStatus.STATUS_CANCELED: - store_job_result(job_id, status, return_info, {}) - else: - store_job_result(job_id, status, return_info, result_data) - except ImportError: - pass # controller 模块可能未加载 - - # 发布状态到桥接器 - if job_id: - for bridge in self.bridges: - if hasattr(bridge, "publish_job_status"): - if goal_status == GoalStatus.STATUS_CANCELED: - bridge.publish_job_status({}, item, status, return_info) - else: - bridge.publish_job_status(result_data, item, status, return_info) + self._finish_error_handled_job( + item, + status, + return_info, + terminal_result_data, + ) except Exception as e: self.lab_logger().error( @@ -1072,16 +1681,12 @@ def get_result_callback(self, item: "QueueItem", action_id: str, future) -> None self.lab_logger().error(traceback.format_exc()) - # 清理 _goals 中的记录 - if job_id in self._goals: - del self._goals[job_id] - - # 发布失败状态 - for bridge in self.bridges: - if hasattr(bridge, "publish_job_status"): - bridge.publish_job_status( - {}, item, "failed", serialize_result_info(f"Callback error: {str(e)}", False, {}) - ) + self._finish_error_handled_job( + item, + "failed", + serialize_result_info(f"Callback error: {str(e)}", False, {}), + {}, + ) def cancel_goal(self, goal_uuid: str) -> bool: """ @@ -1128,6 +1733,15 @@ def get_goal_status(self, job_id: str) -> int: status = g.status self.lab_logger().debug(f"[Host Node] Goal status for {job_id}: {status}") return status + with self._pending_action_error_decisions_lock: + if any( + pending.get("job_id") == job_id + for pending in self._pending_action_error_decisions.values() + ): + return GoalStatus.STATUS_EXECUTING + # retry/fallback 已受理但 ROS goal response 尚未回调时，仍保持执行中投影。 + if job_id in self._error_execution_contexts: + return GoalStatus.STATUS_EXECUTING self.lab_logger().warning(f"[Host Node] Goal {job_id} not found, status unknown") return GoalStatus.STATUS_UNKNOWN diff --git a/unilabos/utils/type_check.py b/unilabos/utils/type_check.py index 5477eca39..7e7456112 100644 --- a/unilabos/utils/type_check.py +++ b/unilabos/utils/type_check.py @@ -1,10 +1,12 @@ import collections.abc import json from collections import OrderedDict -from typing import get_origin, get_args +from typing import Optional, get_origin, get_args import yaml +from unilabos.registry.action_policy import SUCCESS_TYPE_NORMAL, SuccessType + def get_type_class(type_hint): origin = get_origin(type_hint) @@ -68,7 +70,13 @@ def default(self, obj): return str(obj) -def get_result_info_str(error: str, suc: bool, return_value=None) -> str: +def get_result_info_str( + error: str, + suc: bool, + return_value=None, + suc_type: Optional[SuccessType] = None, + error_info: Optional[dict] = None, +) -> str: """ 序列化任务执行结果信息 @@ -86,12 +94,21 @@ def get_result_info_str(error: str, suc: bool, return_value=None) -> str: # if "samples" in return_value and type(return_value["samples"]) in [list, tuple] and type(return_value["samples"][0]) == dict: # samples = return_value.pop("samples") result_info = {"error": error, "suc": suc, "return_value": return_value} + if suc: + result_info["suc_type"] = suc_type or SUCCESS_TYPE_NORMAL + elif error_info: + result_info["error_info"] = error_info return json.dumps(result_info, ensure_ascii=False, cls=ResultInfoEncoder) - -def serialize_result_info(error: str, suc: bool, return_value=None) -> dict: +def serialize_result_info( + error: str, + suc: bool, + return_value=None, + suc_type: Optional[SuccessType] = None, + error_info: Optional[dict] = None, +) -> dict: """ 序列化任务执行结果信息 @@ -104,5 +121,9 @@ def serialize_result_info(error: str, suc: bool, return_value=None) -> dict: JSON字符串格式的结果信息 """ result_info = {"error": error, "suc": suc, "return_value": return_value} + if suc: + result_info["suc_type"] = suc_type or SUCCESS_TYPE_NORMAL + elif error_info: + result_info["error_info"] = error_info return json.loads(json.dumps(result_info, ensure_ascii=False, cls=ResultInfoEncoder)) From 73845ff80d106d3034dab419788af15e765482b8 Mon Sep 17 00:00:00 2001 From: Xuwznln <18435084+Xuwznln@users.noreply.github.com> Date: Sat, 15 Aug 2026 16:52:31 +0800 Subject: [PATCH 2/3] fix(action): complete retry and cancellation lifecycle --- .../action_error_decision_frontend.md | 25 ++ tests/test_action_policy.py | 269 +++++++++++++++++- unilabos/app/communication.py | 5 + unilabos/app/ws_client.py | 12 + unilabos/registry/action_policy.py | 12 + unilabos/ros/nodes/base_device_node.py | 85 +++++- unilabos/ros/nodes/presets/host_node.py | 243 +++++++++++++--- unilabos/utils/exception.py | 4 + 8 files changed, 605 insertions(+), 50 deletions(-) diff --git a/docs/developer_guide/action_error_decision_frontend.md b/docs/developer_guide/action_error_decision_frontend.md index e1c2b757b..a53d7bc3a 100644 --- a/docs/developer_guide/action_error_decision_frontend.md +++ b/docs/developer_guide/action_error_decision_frontend.md @@ -50,6 +50,7 @@ Frontend Host microbackend HostNode Device ```text Device --suc:false--> HostNode --job_error_decision_required--> Cloud Backend Device <--goal-------- HostNode <--job_error_decision----------- Cloud Backend + HostNode --job_error_decision_resolved--> Cloud Backend Cloud Backend <------------------job_status--------------------- HostNode ``` @@ -134,6 +135,8 @@ Cloud Backend <------------------job_status--------------------- HostNode 前端必须以 `option.action` 为稳定值，`label/description` 只用于展示。 `fallback_action` 是只读说明，浏览器不得调用其中的设备动作或修改参数。 +当 `retry_count >= max_retries` 或原动作上下文不可用时，Host 会从 `options` 中移除 +`retry`；前端应直接按最新报告渲染，不得重新补出“重试”按钮。 ## 5. REST 示例 @@ -459,6 +462,28 @@ Host → Backend： } ``` +Host 采用人工选择、执行超时默认动作或取消等待中的 job 后，还会发送终态审计： + +```json +{ + "action": "job_error_decision_resolved", + "data": { + "decision_id": "8a714f4c-5bb0-47b7-9245-9ddf907ef8d4", + "job_id": "df958dcb-b2bf-4a48-94a2-81410bf95a6b", + "task_id": "3f39b087-aec2-4b76-b31d-a3da277e7ec1", + "device_id": "pump-1", + "action_name": "transfer", + "selected_action": "abort", + "reason": "decision_timeout", + "resolved_at": 1786440300.0 + } +} +``` + +`reason` 由操作来源决定：人工提交时透传后端给出的说明；自动超时为 +`decision_timeout`；工作流/job 取消为 `job_canceled`。云后端收到 resolved 后应按 +`decision_id` 幂等移除 pending，并继续等待原 `job_id` 的 `job_status` 终态。 + Backend → Host： ```json diff --git a/tests/test_action_policy.py b/tests/test_action_policy.py index 0a26aea67..78ee34715 100644 --- a/tests/test_action_policy.py +++ b/tests/test_action_policy.py @@ -4,7 +4,7 @@ import pytest -from unilabos.app.ws_client import MessageProcessor, QueueItem +from unilabos.app.ws_client import MessageProcessor, QueueItem, WebSocketClient from unilabos.registry.action_policy import ( ERROR_DECISION_TARGET_BACKEND, ERROR_DECISION_TARGET_MICRO_BACKEND, @@ -19,6 +19,10 @@ _extract_class_body, ) from unilabos.registry.decorators import action, get_action_meta +from unilabos.ros.nodes.base_device_node import ( + _coerce_device_error_info, + _native_driver_result_failed, +) from unilabos.ros.nodes.presets.host_node import HostNode from unilabos.utils.type_check import ( get_result_info_str, @@ -215,6 +219,41 @@ def test_policy_rejects_boolean_numeric_settings(field, value): normalize_error_policy(policy) +def test_policy_requires_timeout_action_for_every_exception_class(): + with pytest.raises(ValueError, match="每个异常 options"): + normalize_error_policy( + { + "options": { + "CommunicationError": [ + {"action": "retry", "label": "重试"}, + ], + "ValueError": [ + {"action": "abort", "label": "终止"}, + ], + }, + "default_on_decision_timeout": "retry", + } + ) + + +def test_policy_accepts_timeout_action_present_for_every_exception_class(): + policy = normalize_error_policy( + { + "options": { + "CommunicationError": [ + {"action": "skip", "label": "跳过"}, + ], + "ValueError": [ + {"action": "skip", "label": "跳过"}, + ], + }, + "default_on_decision_timeout": "skip", + } + ) + + assert policy["default_on_decision_timeout"] == "skip" + + def test_failed_result_carries_structured_error_info(): error_info = { "exception_type": "CommunicationError", @@ -236,6 +275,65 @@ def test_failed_result_carries_structured_error_info(): assert "suc_type" not in encoded +def test_native_action_failure_is_not_confused_with_json_command_boolean_data(): + class HeatChill: + pass + + class UniLabJsonCommand: + pass + + assert _native_driver_result_failed("heat_chill", HeatChill, False) + assert _native_driver_result_failed( + "set_position", HeatChill, {"success": False} + ) + assert not _native_driver_result_failed( + "heat_chill", HeatChill, {"success": True} + ) + assert not _native_driver_result_failed( + "auto-is_empty", UniLabJsonCommand, False + ) + assert not _native_driver_result_failed( + "_execute_driver_command", HeatChill, False + ) + + +def test_native_false_result_gets_structured_action_result_error(): + error_info = _coerce_device_error_info( + "heat_chill", + False, + "driver returned an unsuccessful native action result: False", + ) + + assert error_info["action_name"] == "heat_chill" + assert error_info["exception_type"] == "ActionResultError" + assert error_info["exception_mro"][:2] == ["ActionResultError", "RuntimeError"] + assert "unsuccessful native action result" in error_info["error_message"] + + +def test_native_structured_failure_preserves_driver_error_classification(): + error_info = _coerce_device_error_info( + "set_position", + { + "success": False, + "error": "top-level fallback", + "error_info": { + "exception_type": "CommunicationError", + "exception_mro": ["CommunicationError", "Exception"], + "error_message": "serial port closed", + "category": "communication", + "severity": "recoverable", + }, + }, + "native result failed", + ) + + assert error_info["exception_type"] == "CommunicationError" + assert error_info["exception_mro"] == ["CommunicationError", "Exception"] + assert error_info["error_message"] == "serial port closed" + assert error_info["category"] == "communication" + assert error_info["severity"] == "recoverable" + + class _Logger: def info(self, message): pass @@ -247,11 +345,42 @@ def warning(self, message): class _DecisionBridge: def __init__(self): self.reports = [] + self.resolved_reports = [] def publish_job_error_decision_required(self, report): self.reports.append(report) return True + def publish_job_error_decision_resolved(self, report): + self.resolved_reports.append(report) + return True + + +def test_ws_resolved_error_decision_uses_documented_envelope(): + class _Processor: + def __init__(self): + self.messages = [] + + def send_message(self, message): + self.messages.append(message) + return True + + class _Client: + is_disabled = False + message_processor = _Processor() + + @staticmethod + def is_connected(): + return True + + client = _Client() + report = {"decision_id": "d-1", "selected_action": "abort"} + + assert WebSocketClient.publish_job_error_decision_resolved(client, report) + assert client.message_processor.messages == [ + {"action": "job_error_decision_resolved", "data": report} + ] + class FakeHostDecisionNode: _begin_action_error_decision = HostNode._begin_action_error_decision @@ -260,6 +389,11 @@ class FakeHostDecisionNode: HostNode._handle_action_error_decision_timeout ) handle_action_error_decision = HostNode.handle_action_error_decision + _publish_action_error_decision_resolved = ( + HostNode._publish_action_error_decision_resolved + ) + _request_goal_cancel = HostNode._request_goal_cancel + cancel_job = HostNode.cancel_job get_pending_action_error_decisions = ( HostNode.get_pending_action_error_decisions ) @@ -272,6 +406,7 @@ def __init__(self): self._goals = {"job-1": object()} self._pending_action_error_decisions = {} self._pending_action_error_decisions_lock = threading.RLock() + self._canceled_jobs = set() self._error_execution_contexts = { "job-1": { "item": _queue_item(), @@ -321,6 +456,8 @@ def _finish_error_handled_job( ): self.finished.append((item, status, return_info, result_data)) self._error_execution_contexts.pop(item.job_id, None) + self._goals.pop(item.job_id, None) + self._canceled_jobs.discard(item.job_id) def _queue_item( @@ -649,9 +786,38 @@ def test_host_decision_validates_identity_and_first_result_wins(): ) assert host.finished[0][1] == "success" assert host.finished[0][2]["suc_type"] == SUCCESS_TYPE_SKIP + assert host.bridge.resolved_reports[0]["selected_action"] == "skip" def test_host_retry_limit_fails_closed(): + host = FakeHostDecisionNode() + host._error_execution_contexts["job-1"]["retry_count"] = 1 + host._action_value_mappings["device-1"]["run"]["error_policy"] = ( + normalize_error_policy( + { + "options": { + "CommunicationError": [ + {"action": "retry", "label": "重试"} + ] + }, + "max_retries": 1, + } + ) + ) + + return_info = _error_return_info() + assert not host._begin_action_error_decision( + _queue_item(), + return_info, + {"return_info": "failed"}, + ) + assert not host.sent_goals + assert not host._pending_action_error_decisions + assert not host.bridge.reports + assert "达到最大重试次数（1）" in return_info["error"] + + +def test_host_retry_exhaustion_hides_retry_and_timeout_falls_back_to_abort(): host = FakeHostDecisionNode() host._error_execution_contexts["job-1"]["retry_count"] = 1 decision_id = _begin_pending( @@ -659,22 +825,113 @@ def test_host_retry_limit_fails_closed(): { "options": { "CommunicationError": [ - {"action": "retry", "label": "重试"} + {"action": "retry", "label": "重试"}, + {"action": "abort", "label": "终止"}, ] }, "max_retries": 1, + "default_on_decision_timeout": "retry", }, ) - assert host.handle_action_error_decision( + report = host.bridge.reports[0] + assert [option["action"] for option in report["options"]] == ["abort"] + assert "达到最大重试次数（1）" in report["error_message"] + assert report["default_on_decision_timeout"] == "abort" + + host._handle_action_error_decision_timeout(decision_id) + + assert host.finished[0][1] == "failed" + assert host.bridge.resolved_reports[0]["selected_action"] == "abort" + assert host.bridge.resolved_reports[0]["reason"] == "decision_timeout" + + +def test_host_without_execution_context_does_not_offer_retry(): + host = FakeHostDecisionNode() + host._error_execution_contexts.clear() + + assert host._begin_action_error_decision( + _queue_item(), + _error_return_info(), + {"return_info": "failed"}, + ) + + report = host.bridge.reports[0] + assert [option["action"] for option in report["options"]] == [ + "skip", + "abort", + ] + assert "缺少原动作上下文，无法重试" in report["error_message"] + + +def test_cancel_pending_error_decision_closes_timer_and_rejects_late_reply(): + host = FakeHostDecisionNode() + decision_id = _begin_pending(host) + timer = host._pending_action_error_decisions[decision_id]["timer"] + + assert host.cancel_job("job-1") + + timer.join(timeout=1) + assert not timer.is_alive() + assert not host._pending_action_error_decisions + assert not host._error_execution_contexts + assert host.finished[0][1] == "failed" + assert host.bridge.resolved_reports[0]["selected_action"] == "cancel" + assert host.bridge.resolved_reports[0]["reason"] == "job_canceled" + assert not host.handle_action_error_decision( decision_id, "job-1", {"action": "retry"}, ) - assert not host.sent_goals - assert host.finished[0][1] == "failed" - assert "exceeded 1 retries" in host.finished[0][2]["error"] + +def test_goal_accepted_after_inflight_cancel_is_canceled_immediately(): + class _CancelFuture: + def add_done_callback(self, callback): + self.callback = callback + + class _ResultFuture: + def add_done_callback(self, callback): + self.callback = callback + + def result(self): + raise AssertionError("canceled goal response must not block on result") + + class _GoalHandle: + accepted = True + + def __init__(self): + self.cancel_calls = 0 + self.result_future = _ResultFuture() + + def get_result_async(self): + return self.result_future + + def cancel_goal_async(self): + self.cancel_calls += 1 + return _CancelFuture() + + class _GoalResponseFuture: + def __init__(self, goal_handle): + self.goal_handle = goal_handle + + def result(self): + return self.goal_handle + + host = FakeHostDecisionNode() + host._goals.clear() + host._canceled_jobs.add("job-1") + goal_handle = _GoalHandle() + + HostNode.goal_response_callback( + host, + _queue_item(), + "/devices/device-1/run", + _GoalResponseFuture(goal_handle), + ) + + assert host._goals["job-1"] is goal_handle + assert goal_handle.cancel_calls == 1 def test_host_dispatches_registered_fallback_action(): diff --git a/unilabos/app/communication.py b/unilabos/app/communication.py index 695e21509..63f87ecc5 100644 --- a/unilabos/app/communication.py +++ b/unilabos/app/communication.py @@ -96,6 +96,11 @@ def publish_action_locks(self, locks: list) -> None: """ pass + def publish_job_error_decision_resolved(self, report: dict) -> bool: + """上报自动或人工异常决策的最终选择；不支持时返回 False。""" + + return False + def setup_pong_subscription(self) -> None: """ 设置pong消息订阅（可选实现） diff --git a/unilabos/app/ws_client.py b/unilabos/app/ws_client.py index ce1170de1..aa98ab647 100644 --- a/unilabos/app/ws_client.py +++ b/unilabos/app/ws_client.py @@ -1734,6 +1734,18 @@ def publish_job_error_decision_required(self, report: Dict[str, Any]) -> bool: ) return queued + def publish_job_error_decision_resolved(self, report: Dict[str, Any]) -> bool: + """向云端上报 Host 已采用的异常决策，供 timeout/人工操作审计。""" + + if self.is_disabled or not self.is_connected(): + logger.warning( + "[WebSocketClient] Not connected, cannot report resolved error decision" + ) + return False + return self.message_processor.send_message( + {"action": "job_error_decision_resolved", "data": report} + ) + def send_ping(self, ping_id: str, timestamp: float) -> None: """发送ping消息""" if self.is_disabled or not self.is_connected(): diff --git a/unilabos/registry/action_policy.py b/unilabos/registry/action_policy.py index b7e9eefe2..753111bc8 100644 --- a/unilabos/registry/action_policy.py +++ b/unilabos/registry/action_policy.py @@ -136,6 +136,18 @@ def normalize_error_policy( timeout_action = policy.get("default_on_decision_timeout", "abort") if timeout_action not in {"abort", "retry", "skip"}: raise ValueError("default_on_decision_timeout 仅支持 abort/retry/skip") + if timeout_action != "abort": + missing = [ + error_class_name + for error_class_name, class_options in options.items() + if timeout_action + not in {str(option.get("action")) for option in class_options} + ] + if missing: + raise ValueError( + "default_on_decision_timeout 必须存在于每个异常 options 中；" + f"缺少 {timeout_action!r}: {missing}" + ) normalized["default_on_decision_timeout"] = timeout_action return normalized diff --git a/unilabos/ros/nodes/base_device_node.py b/unilabos/ros/nodes/base_device_node.py index edaf5ebdc..8726c2ff7 100644 --- a/unilabos/ros/nodes/base_device_node.py +++ b/unilabos/ros/nodes/base_device_node.py @@ -81,7 +81,7 @@ from unilabos.utils.import_manager import default_manager from unilabos.utils.log import info, debug, warning, error, critical, logger, trace from unilabos.utils.type_check import get_type_class, TypeEncoder, get_result_info_str -from unilabos.utils.exception import DeviceActionError +from unilabos.utils.exception import ActionResultError, DeviceActionError if TYPE_CHECKING: from pylabrobot.resources import Resource as ResourcePLR @@ -89,6 +89,74 @@ T = TypeVar("T") +def _native_driver_result_failed( + action_name: str, action_type: Any, value: Any +) -> bool: + """原生 ROS Action 的 bool/dict success 是业务成功位；JSON Command 可返回 bool 数据。""" + + type_name = str(getattr(action_type, "__name__", "")) + if action_name.startswith("_execute_driver_command") or type_name.startswith( + "UniLabJsonCommand" + ): + return False + if value is False: + return True + return isinstance(value, dict) and value.get("success") is False + + +def _coerce_device_error_info( + action_name: str, + value: Any, + error_text: str, +) -> Dict[str, Any]: + """把原生 Action 的失败返回归一化为 Host 可匹配的结构化错误。""" + + source: Dict[str, Any] = {} + if isinstance(value, dict): + provided = value.get("error_info") + source = ( + {**value, **provided} + if isinstance(provided, dict) + else dict(value) + ) + + exception_type = str(source.get("exception_type") or "ActionResultError") + raw_mro = source.get("exception_mro") + if isinstance(raw_mro, list) and raw_mro: + exception_mro = [str(name) for name in raw_mro] + elif exception_type == "ActionResultError": + exception_mro = [ + error_class.__name__ for error_class in ActionResultError.__mro__ + ] + else: + exception_mro = [ + exception_type, + "Exception", + "BaseException", + "object", + ] + + error_message = str( + source.get("error_message") + or source.get("error") + or source.get("message") + or source.get("reason") + or error_text + or "device action reported an unsuccessful result" + ) + error_info: Dict[str, Any] = { + "action_name": str(source.get("action_name") or action_name), + "exception_type": exception_type, + "exception_mro": exception_mro, + "error_message": error_message, + "traceback": str(source.get("traceback") or error_text or error_message), + } + for key in ("category", "severity"): + if source.get(key) is not None: + error_info[key] = str(source[key]) + return error_info + + class RclpyAsyncMutex: """rclpy executor 兼容的异步互斥锁 @@ -2253,8 +2321,15 @@ async def _wake(): execution_success = False action_return_value = _raw_result elif not execution_error: - execution_success = True action_return_value = _raw_result + execution_success = not _native_driver_result_failed( + action_name, action_type, _raw_result + ) + if not execution_success: + execution_error = ( + "driver returned an unsuccessful native action result: " + f"{_raw_result!r}" + ) if isinstance(_raw_result, BaseException): execution_error_info = { @@ -2277,6 +2352,12 @@ async def _wake(): execution_error_info["severity"] = str( getattr(severity, "value", severity) ) + elif not execution_success: + execution_error_info = _coerce_device_error_info( + report_action_name, + _raw_result, + execution_error, + ) # 清理 feedback timer if _feedback_timer is not None: diff --git a/unilabos/ros/nodes/presets/host_node.py b/unilabos/ros/nodes/presets/host_node.py index 8afffebee..563db1713 100644 --- a/unilabos/ros/nodes/presets/host_node.py +++ b/unilabos/ros/nodes/presets/host_node.py @@ -372,6 +372,8 @@ def __init__( self._error_execution_contexts: Dict[str, Dict[str, Any]] = {} self._pending_action_error_decisions: Dict[str, Dict[str, Any]] = {} self._pending_action_error_decisions_lock = threading.RLock() + # cancel 可能发生在 Goal 等待响应、执行中或等待异常决策三个阶段。 + self._canceled_jobs: Set[str] = set() self._online_devices: Set[str] = {f"{self.namespace}/{device_id}"} # 用于跟踪在线设备 self._last_discovery_time = 0.0 # 上次设备发现的时间 self._discovery_lock = threading.Lock() # 设备发现的互斥锁 @@ -884,6 +886,12 @@ def send_goal( server_info: 服务器发送信息，包含发送时间戳等 """ callback_item = result_item or item + with self._pending_action_error_decisions_lock: + if callback_item.job_id in self._canceled_jobs: + self.lab_logger().info( + f"[Host Node] Skip canceled goal {callback_item.job_id[:8]}" + ) + return u = uuid.UUID(transport_goal_id or item.job_id) device_id = item.device_id action_name = item.action_name @@ -1085,6 +1093,7 @@ def _finish_error_handled_job( timer = pending.get("timer") if timer is not None: timer.cancel() + self._canceled_jobs.discard(job_id) try: from unilabos.app.web.controller import store_job_result @@ -1119,6 +1128,10 @@ def _begin_action_error_decision( ) -> bool: """设备失败后在 Host 创建决策点；成功上报时原 job 继续保持 pending。""" + with self._pending_action_error_decisions_lock: + if item.job_id in self._canceled_jobs: + return False + raw_error_info = return_info.get("error_info") if not isinstance(raw_error_info, dict): return False @@ -1148,16 +1161,65 @@ def _begin_action_error_decision( options = resolve_error_options_by_names(policy, exception_mro) if not options: return False + execution_context = self._error_execution_contexts.get(item.job_id) + retry_count = int((execution_context or {}).get("retry_count", 0)) + max_retries = int(policy.get("max_retries", 3)) + has_retry_option = any( + str(option.get("action")) == "retry" for option in options + ) + retry_unavailable_message = "" + if has_retry_option and execution_context is None: + options = [ + option + for option in options + if str(option.get("action")) != "retry" + ] + retry_unavailable_message = "缺少原动作上下文，无法重试" + elif has_retry_option and retry_count >= max_retries: + options = [ + option + for option in options + if str(option.get("action")) != "retry" + ] + retry_unavailable_message = f"达到最大重试次数（{max_retries}）" + + if retry_unavailable_message: + previous_message = str( + raw_error_info.get("error_message") + or return_info.get("error") + or "" + ) + raw_error_info = { + **raw_error_info, + "error_message": ( + f"{previous_message}\n{retry_unavailable_message}" + if previous_message + else retry_unavailable_message + ), + } + return_info["error"] = raw_error_info["error_message"] + return_info["error_info"] = dict(raw_error_info) + if "return_info" in result_data: + result_data["return_info"] = json.dumps( + return_info, + ensure_ascii=False, + ) + # 重试是唯一选项且已不可用时，不创建无法处理的决策点，直接失败。 + if not options: + return False + timeout_action = str(policy.get("default_on_decision_timeout", "abort")) + if timeout_action != "abort" and timeout_action not in { + str(option.get("action")) for option in options + }: + timeout_action = "abort" error_info = { **raw_error_info, "options": options, - "max_retries": int(policy.get("max_retries", 3)), + "max_retries": max_retries, "decision_timeout_seconds": float( policy.get("decision_timeout_seconds", 300.0) ), - "default_on_decision_timeout": str( - policy.get("default_on_decision_timeout", "abort") - ), + "default_on_decision_timeout": timeout_action, } decision_target = str( getattr(item, "error_decision_target", ERROR_DECISION_TARGET_BACKEND) @@ -1171,7 +1233,6 @@ def _begin_action_error_decision( ) decision_target = ERROR_DECISION_TARGET_BACKEND - execution_context = self._error_execution_contexts.get(item.job_id) if execution_context is None: self.lab_logger().warning( f"[Host Node] Job {item.job_id[:8]} 缺少重试上下文，仅支持 skip/abort" @@ -1206,7 +1267,7 @@ def _begin_action_error_decision( "error_message": error_info.get("error_message", return_info.get("error", "")), "traceback": error_info.get("traceback", return_info.get("error", "")), "options": options, - "retry_count": int((execution_context or {}).get("retry_count", 0)), + "retry_count": retry_count, "max_retries": int(error_info.get("max_retries", 3)), "created_at": created_at, "decision_timeout_seconds": timeout_seconds, @@ -1290,6 +1351,48 @@ def get_pending_action_error_decisions( ] return reports + def _publish_action_error_decision_resolved( + self, + pending: Dict[str, Any], + selected_action: str, + reason: str = "", + ) -> Dict[str, Any]: + """发布决策终态审计；观测/通信失败不影响实际决策。""" + + item = pending["item"] + resolved_report = { + "decision_id": pending["decision_id"], + "job_id": pending["job_id"], + "task_id": item.task_id, + "device_id": item.device_id, + "action_name": item.action_name, + "selected_action": selected_action, + "reason": reason, + "resolved_at": time.time(), + } + self._emit_local_action_event( + item, + "job_error_decision_resolved", + resolved_report, + ) + if pending.get("decision_target") == ERROR_DECISION_TARGET_BACKEND: + for bridge in self.bridges: + publish_resolved = getattr( + bridge, + "publish_job_error_decision_resolved", + None, + ) + if not callable(publish_resolved): + continue + try: + if publish_resolved(deepcopy(resolved_report)): + break + except Exception as ex: # noqa: BLE001 - 审计上报失败不阻断决策 + self.lab_logger().warning( + f"[Host Node] 异常决策结果上报失败: {bridge!r}: {ex}" + ) + return resolved_report + def _handle_action_error_decision_timeout(self, decision_id: str) -> None: with self._pending_action_error_decisions_lock: pending = self._pending_action_error_decisions.get(decision_id) @@ -1297,13 +1400,22 @@ def _handle_action_error_decision_timeout(self, decision_id: str) -> None: return error_info = pending["error_info"] job_id = pending["job_id"] + timeout_action = str( + error_info.get("default_on_decision_timeout", "abort") + ) + allowed_actions = { + str(option.get("action")) + for option in error_info.get("options", []) + } + if timeout_action != "abort" and timeout_action not in allowed_actions: + timeout_action = "abort" self.handle_action_error_decision( decision_id, job_id, { "decision_id": decision_id, "job_id": job_id, - "action": error_info.get("default_on_decision_timeout", "abort"), + "action": timeout_action, "reason": "decision_timeout", }, ) @@ -1327,7 +1439,11 @@ def handle_action_error_decision( if candidate.get("job_id") == job_id ] pending = matches[0] if len(matches) == 1 else None - if pending is None or pending.get("resolving"): + if ( + pending is None + or pending.get("resolving") + or pending.get("job_id") in self._canceled_jobs + ): return False if ( decision_target is not None @@ -1359,7 +1475,10 @@ def handle_action_error_decision( (candidate for candidate in options if str(candidate.get("action")) == selected), None, ) - if option is None and decision.get("reason") != "decision_timeout": + if option is None and not ( + decision.get("reason") == "decision_timeout" + and selected == "abort" + ): return False pending["resolving"] = True @@ -1369,19 +1488,10 @@ def handle_action_error_decision( timer.cancel() item = pending["item"] - self._emit_local_action_event( - item, - "job_error_decision_resolved", - { - "decision_id": pending["decision_id"], - "job_id": pending["job_id"], - "task_id": item.task_id, - "device_id": item.device_id, - "action_name": item.action_name, - "selected_action": selected, - "reason": str(decision.get("reason") or ""), - "resolved_at": time.time(), - }, + self._publish_action_error_decision_resolved( + pending, + selected, + str(decision.get("reason") or ""), ) if selected == "abort": self._finish_error_handled_job( @@ -1566,6 +1676,14 @@ def goal_response_callback( recovery_suc_type=recovery_suc_type, ) ) + with self._pending_action_error_decisions_lock: + canceled = item.job_id in self._canceled_jobs + if canceled: + self.lab_logger().info( + f"[Host Node] Goal {item.job_id[:8]} accepted after cancel; cancel immediately" + ) + self._request_goal_cancel(item.job_id, goal_handle) + return goal_future.result() def feedback_callback(self, item: "QueueItem", action_id: str, feedback_msg) -> None: @@ -1597,9 +1715,11 @@ def get_result_callback( result = future.result() result_msg = result.result goal_status = result.status + with self._pending_action_error_decisions_lock: + cancel_requested = job_id in self._canceled_jobs # 检查是否是被取消的任务 - if goal_status == GoalStatus.STATUS_CANCELED: + if cancel_requested or goal_status == GoalStatus.STATUS_CANCELED: self.lab_logger().info(f"[Host Node] Goal {action_id} ({job_id[:8]}) was cancelled") status = "failed" return_info = serialize_result_info("Job was cancelled", False, {}) @@ -1647,7 +1767,7 @@ def get_result_callback( ) terminal_result_data = ( - {} if goal_status == GoalStatus.STATUS_CANCELED else result_data + {} if cancel_requested or goal_status == GoalStatus.STATUS_CANCELED else result_data ) if ( status == "failed" @@ -1664,7 +1784,7 @@ def get_result_callback( return self.lab_logger().info(f"[Host Node] Result for {action_id} ({job_id[:8]}): {status}") - if goal_status != GoalStatus.STATUS_CANCELED: + if not cancel_requested and goal_status != GoalStatus.STATUS_CANCELED: self.lab_logger().trace(f"[Host Node] Result data: {result_data}") self._finish_error_handled_job( item, @@ -1688,29 +1808,68 @@ def get_result_callback( {}, ) - def cancel_goal(self, goal_uuid: str) -> bool: - """ - 取消目标 + def _request_goal_cancel(self, job_id: str, goal_handle: Any) -> None: + """向已受理的 ROS Goal 发起取消。""" - Args: - goal_uuid: 目标UUID（job_id） + cancel_future = goal_handle.cancel_goal_async() + cancel_future.add_done_callback( + lambda future: self._cancel_goal_callback(job_id, future) + ) - Returns: - bool: 如果找到目标并发起取消请求返回True，否则返回False - """ - if goal_uuid in self._goals: - self.lab_logger().info(f"[Host Node] Cancelling goal {goal_uuid[:8]}") - goal_handle = self._goals[goal_uuid] + def cancel_job(self, job_id: str) -> bool: + """取消运行中、等待 Goal 响应或等待异常决策的逻辑 job。""" - # 发起异步取消请求 - cancel_future = goal_handle.cancel_goal_async() + pending = None + with self._pending_action_error_decisions_lock: + for decision_id, candidate in list( + self._pending_action_error_decisions.items() + ): + if candidate.get("job_id") != job_id: + continue + pending = self._pending_action_error_decisions.pop(decision_id) + pending["resolving"] = True + timer = pending.get("timer") + if timer is not None: + timer.cancel() + break - # 添加取消完成的回调 - cancel_future.add_done_callback(lambda future: self._cancel_goal_callback(goal_uuid, future)) + goal_handle = self._goals.get(job_id) + has_context = job_id in self._error_execution_contexts + if pending is None and goal_handle is None and not has_context: + self.lab_logger().warning( + f"[Host Node] Job {job_id[:8]} not found, cannot cancel" + ) + return False + self._canceled_jobs.add(job_id) + + if pending is not None: + item = pending["item"] + self._publish_action_error_decision_resolved( + pending, + "cancel", + "job_canceled", + ) + self._finish_error_handled_job( + item, + "failed", + serialize_result_info("Job was cancelled", False, {}), + {}, + ) return True + + if goal_handle is not None: + self.lab_logger().info(f"[Host Node] Cancelling goal {job_id[:8]}") + self._request_goal_cancel(job_id, goal_handle) else: - self.lab_logger().warning(f"[Host Node] Goal {goal_uuid[:8]} not found in _goals, cannot cancel") - return False + self.lab_logger().info( + f"[Host Node] Marked in-flight goal {job_id[:8]} canceled before acceptance" + ) + return True + + def cancel_goal(self, goal_uuid: str) -> bool: + """兼容旧接口；统一走可覆盖异常决策和重试在途状态的取消逻辑。""" + + return self.cancel_job(goal_uuid) def _cancel_goal_callback(self, goal_uuid: str, future) -> None: """取消目标的回调""" diff --git a/unilabos/utils/exception.py b/unilabos/utils/exception.py index b98ecaf62..c688e4f43 100644 --- a/unilabos/utils/exception.py +++ b/unilabos/utils/exception.py @@ -34,3 +34,7 @@ def __init__( detail = " (目标拒绝了请求)" if rejected else "" suffix = f": {self.remote_error}" if self.remote_error else "" super().__init__(f"调用设备动作 [{device_id}.{action_name}] 失败{detail}{suffix}") + + +class ActionResultError(RuntimeError): + """设备未抛异常、但通过原生 Action 结果明确报告失败。""" From f5c081e38a190358f2b38e70ff57c0129b72f891 Mon Sep 17 00:00:00 2001 From: Xuwznln <18435084+Xuwznln@users.noreply.github.com> Date: Sat, 15 Aug 2026 19:58:56 +0800 Subject: [PATCH 3/3] fix(action): harden decision expiry and replay --- .../action_error_decision_frontend.md | 43 ++- tests/test_action_policy.py | 216 +++++++++++++-- unilabos/app/model.py | 3 + unilabos/app/web/api.py | 12 +- unilabos/app/web/controller.py | 35 ++- unilabos/app/ws_client.py | 48 +++- unilabos/ros/nodes/presets/host_node.py | 257 +++++++++++++++--- 7 files changed, 546 insertions(+), 68 deletions(-) diff --git a/docs/developer_guide/action_error_decision_frontend.md b/docs/developer_guide/action_error_decision_frontend.md index a53d7bc3a..77b2262f1 100644 --- a/docs/developer_guide/action_error_decision_frontend.md +++ b/docs/developer_guide/action_error_decision_frontend.md @@ -178,7 +178,13 @@ Accept: application/json POST /api/v1/error-decisions/8a714f4c-5bb0-47b7-9245-9ddf907ef8d4 Content-Type: application/json -{"action":"retry","reason":"operator confirmed"} +{ + "decision_id": "8a714f4c-5bb0-47b7-9245-9ddf907ef8d4", + "job_id": "df958dcb-b2bf-4a48-94a2-81410bf95a6b", + "device_id": "pump-1", + "action": "retry", + "reason": "operator confirmed" +} ``` 成功只表示 Host 接受了命令，不表示恢复动作已经成功： @@ -194,6 +200,9 @@ Content-Type: application/json ```json { + "decision_id": "8a714f4c-5bb0-47b7-9245-9ddf907ef8d4", + "job_id": "df958dcb-b2bf-4a48-94a2-81410bf95a6b", + "device_id": "pump-1", "action": "reset_connection", "reason": "operator selected registered recovery" } @@ -203,6 +212,9 @@ Content-Type: application/json ```json { + "decision_id": "8a714f4c-5bb0-47b7-9245-9ddf907ef8d4", + "job_id": "df958dcb-b2bf-4a48-94a2-81410bf95a6b", + "device_id": "pump-1", "action": "manual_result", "result": {"confirmed_volume": 10.0}, "reason": "verified on instrument" @@ -211,9 +223,14 @@ Content-Type: application/json 错误语义： -- `404`：不存在、已被其他请求处理、已经超时，或通道来源不匹配。 +- `422`：缺少 `decision_id/job_id/device_id` 中任一项或请求结构非法。 +- `409`：路径与正文身份不一致，或 Host 在 timer 回调前已按 `expires_at` + 原子执行超时默认动作。 +- `404`：不存在或通道来源不匹配。 - `503`：HostNode 尚未就绪。 -- 第一次合法决策获胜；前端收到 `404` 时重新 GET 列表。若列表中已不存在该 ID，关闭弹窗并继续追踪原 job。 +- 第一次合法决策获胜。重复提交同一已解决决策时，Host 在短期 tombstone + 保留窗口内返回 `status=resolved, replayed=true` 和原 `resolution`，但不会再次执行。 + 前端收到 `404/409` 时重新 GET 列表；若列表中已不存在该 ID，关闭弹窗并继续追踪原 job。 ### 5.3 查询原 job @@ -416,13 +433,16 @@ async function resolveDecision( method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ + decision_id: decision.decision_id, + job_id: decision.job_id, + device_id: decision.device_id, action, ...(result === undefined ? {} : { result }), reason: "operator confirmed", }), }, ); - if (response.status === 404) { + if (response.status === 404 || response.status === 409) { await refreshDecisions(); return; } @@ -483,6 +503,12 @@ Host 采用人工选择、执行超时默认动作或取消等待中的 job 后 `reason` 由操作来源决定：人工提交时透传后端给出的说明；自动超时为 `decision_timeout`；工作流/job 取消为 `job_canceled`。云后端收到 resolved 后应按 `decision_id` 幂等移除 pending，并继续等待原 `job_id` 的 `job_status` 终态。 +取消 pending 决策时 `selected_action=abort`，原 job 终态为 `canceled`；设备动作此前 +已经返回失败，因此 Host 不会再次向设备发送 cancel。 + +Host 默认在内存中保留一小时 resolved/expired tombstone，并在云 WS 重连时重放仍在 +窗口内的 required 与 resolved。云后端仍须永久保存决策终态；Host tombstone 只用于 +覆盖短时断线和重复提交，不替代后端审计。 Backend → Host： @@ -521,9 +547,10 @@ tracking_job 2. pending 期间原 job 仍为执行中，不能先标记 failed。 3. POST `delivered` 不是 job 成功，只是 Host 已接受选择。 4. retry 使用新的 ROS transport goal UUID，但前端始终追踪原 `job_id`。 -5. 只允许提交报告 `options` 中的 action；最终合法性仍由 Host 校验。 -6. Host 超时是权威；浏览器倒计时归零后只刷新，不自行执行默认动作。 -7. fallback 由 Host 通过 ActionClient 发给真实设备，浏览器绝不调用设备 Service/Action。 +5. 每次提交必须原样携带 `decision_id/job_id/device_id`，三项完全一致后 Host 才消费。 +6. 只允许提交报告 `options` 中的 action；最终合法性仍由 Host 校验。 +7. Host 超时是权威；浏览器倒计时归零后只刷新，不自行执行默认动作。 +8. fallback 由 Host 通过 ActionClient 发给真实设备，浏览器绝不调用设备 Service/Action。 ## 10. 前端验收清单 @@ -531,7 +558,7 @@ tracking_job - 新异常通过 SSE 在不刷新页面时出现。 - 同一 `decision_id` 的 snapshot/SSE 重复消息只产生一个 UI 项。 - 点击后立即禁用按钮，成功响应后继续追踪原 job。 -- POST 响应丢失后再次提交得到 404，前端通过 snapshot 正确收敛。 +- POST 响应丢失后再次提交得到原 resolved tombstone，前端不会重复执行并继续追踪 job。 - SSE `seq` 出现空洞时重新拉 snapshot。 - timeout、另一个浏览器先处理、云/本地通道错投时不会重复执行。 - retry/fallback 成功后展示 `suc_type`；skip 明确提示需要物料复核。 diff --git a/tests/test_action_policy.py b/tests/test_action_policy.py index 78ee34715..2f61aeb56 100644 --- a/tests/test_action_policy.py +++ b/tests/test_action_policy.py @@ -1,6 +1,7 @@ import asyncio import ast import json +import time import pytest @@ -382,7 +383,54 @@ def is_connected(): ] +def test_ws_reconnect_replays_pending_and_resolved_decisions(monkeypatch): + class _Host: + @staticmethod + def get_pending_action_error_decisions(decision_target=None): + assert decision_target == ERROR_DECISION_TARGET_BACKEND + return [{"decision_id": "pending-1"}] + + @staticmethod + def get_resolved_action_error_decisions(decision_target=None): + assert decision_target == ERROR_DECISION_TARGET_BACKEND + return [{"decision_id": "resolved-1"}] + + class _Client: + is_disabled = False + + def __init__(self): + self.required = [] + self.resolved = [] + + @staticmethod + def is_connected(): + return True + + def publish_job_error_decision_required(self, report): + self.required.append(report) + return True + + def publish_job_error_decision_resolved(self, report): + self.resolved.append(report) + return True + + monkeypatch.setattr( + HostNode, + "get_instance", + classmethod(lambda cls, index=0: _Host()), + ) + client = _Client() + + WebSocketClient.report_action_error_decisions(client) + + assert client.required == [{"decision_id": "pending-1"}] + assert client.resolved == [{"decision_id": "resolved-1"}] + + class FakeHostDecisionNode: + _ACTION_ERROR_DECISION_TOMBSTONE_TTL_SECONDS = ( + HostNode._ACTION_ERROR_DECISION_TOMBSTONE_TTL_SECONDS + ) _begin_action_error_decision = HostNode._begin_action_error_decision _emit_local_action_event = staticmethod(HostNode._emit_local_action_event) _handle_action_error_decision_timeout = ( @@ -392,6 +440,21 @@ class FakeHostDecisionNode: _publish_action_error_decision_resolved = ( HostNode._publish_action_error_decision_resolved ) + _prune_action_error_decision_tombstones_locked = ( + HostNode._prune_action_error_decision_tombstones_locked + ) + _remember_action_error_decision_resolution_locked = ( + HostNode._remember_action_error_decision_resolution_locked + ) + get_resolved_action_error_decision = ( + HostNode.get_resolved_action_error_decision + ) + get_resolved_action_error_decisions = ( + HostNode.get_resolved_action_error_decisions + ) + replay_action_error_decision_resolution = ( + HostNode.replay_action_error_decision_resolution + ) _request_goal_cancel = HostNode._request_goal_cancel cancel_job = HostNode.cancel_job get_pending_action_error_decisions = ( @@ -405,6 +468,7 @@ def __init__(self): self.bridges = [self.bridge] self._goals = {"job-1": object()} self._pending_action_error_decisions = {} + self._resolved_action_error_decisions = {} self._pending_action_error_decisions_lock = threading.RLock() self._canceled_jobs = set() self._error_execution_contexts = { @@ -511,6 +575,23 @@ def _begin_pending(host, policy=None, item=None): return decision_id +def _decision( + decision_id, + action, + *, + job_id="job-1", + device_id="device-1", + **extra, +): + return { + "decision_id": decision_id, + "job_id": job_id, + "device_id": device_id, + "action": action, + **extra, + } + + def test_host_owns_decision_and_publishes_registry_options(): host = FakeHostDecisionNode() decision_id = _begin_pending(host) @@ -532,7 +613,7 @@ def test_host_owns_decision_and_publishes_registry_options(): assert host.handle_action_error_decision( decision_id, "job-1", - {"action": "abort"}, + _decision(decision_id, "abort"), ) @@ -575,6 +656,28 @@ def handle_action_error_decision( ] +@pytest.mark.parametrize("missing", ["decision_id", "job_id", "device_id"]) +def test_ws_decision_requires_complete_identity(monkeypatch, missing): + class _Host: + def handle_action_error_decision(self, *args, **kwargs): + raise AssertionError("incomplete identity must not reach HostNode") + + monkeypatch.setattr( + HostNode, + "get_instance", + classmethod(lambda cls, index=0: _Host()), + ) + payload = { + "decision_id": "decision-ws", + "job_id": "job-ws", + "device_id": "remote-device", + "action": "retry", + } + payload.pop(missing) + + asyncio.run(MessageProcessor._handle_job_error_decision(object(), payload)) + + def test_host_micro_backend_decision_stays_local_and_rejects_cloud_reply(): from unilabos.app.web.event_bus import monitor_bus @@ -599,13 +702,13 @@ def test_host_micro_backend_decision_stays_local_and_rejects_cloud_reply(): assert not host.handle_action_error_decision( decision_id, "job-1", - {"action": "abort"}, + _decision(decision_id, "abort"), decision_target=ERROR_DECISION_TARGET_BACKEND, ) assert host.handle_action_error_decision( decision_id, "job-1", - {"action": "retry"}, + _decision(decision_id, "retry"), decision_target=ERROR_DECISION_TARGET_MICRO_BACKEND, ) resolved_event = event_queue.get(timeout=1) @@ -692,18 +795,32 @@ def test_micro_backend_rest_contract_roundtrip(monkeypatch): assert snapshot.json()["host_ready"] is True assert snapshot.json()["pending_error_decisions"][0]["decision_id"] == decision_id - response = client.post( + incomplete = client.post( f"/api/v1/error-decisions/{decision_id}", json={"action": "skip", "reason": "operator confirmed"}, ) + assert incomplete.status_code == 422 + + request = _decision( + decision_id, + "skip", + reason="operator confirmed", + ) + response = client.post( + f"/api/v1/error-decisions/{decision_id}", + json=request, + ) assert response.status_code == 200 assert response.json() == {"decision_id": decision_id, "status": "delivered"} response = client.post( f"/api/v1/error-decisions/{decision_id}", - json={"action": "skip"}, + json=request, ) - assert response.status_code == 404 + assert response.status_code == 200 + assert response.json()["status"] == "resolved" + assert response.json()["replayed"] is True + assert response.json()["resolution"]["selected_action"] == "skip" store_job_result( "job-poll", @@ -746,7 +863,7 @@ def test_host_retry_uses_existing_action_client_path_and_new_transport_id(): assert host.handle_action_error_decision( decision_id, "job-1", - {"action": "retry"}, + _decision(decision_id, "retry"), ) args, kwargs = host.sent_goals[0] @@ -767,28 +884,91 @@ def test_host_decision_validates_identity_and_first_result_wins(): assert not host.handle_action_error_decision( decision_id, "other-job", - {"action": "retry"}, + _decision(decision_id, "retry", job_id="other-job"), ) assert not host.handle_action_error_decision( decision_id, "job-1", - {"decision_id": "other-decision", "action": "retry"}, + _decision("other-decision", "retry"), ) assert host.handle_action_error_decision( decision_id, "job-1", - {"action": "skip", "result": {"ignored": True}}, + _decision(decision_id, "skip", result={"ignored": True}), ) assert not host.handle_action_error_decision( decision_id, "job-1", - {"action": "abort"}, + _decision(decision_id, "abort"), ) assert host.finished[0][1] == "success" assert host.finished[0][2]["suc_type"] == SUCCESS_TYPE_SKIP assert host.bridge.resolved_reports[0]["selected_action"] == "skip" +@pytest.mark.parametrize("missing", ["decision_id", "job_id", "device_id"]) +def test_host_decision_requires_complete_identity(missing): + host = FakeHostDecisionNode() + decision_id = _begin_pending(host) + decision = _decision(decision_id, "abort") + decision.pop(missing) + + assert not host.handle_action_error_decision( + decision_id, + "job-1", + decision, + ) + assert decision_id in host._pending_action_error_decisions + + +def test_expired_manual_decision_atomically_runs_timeout_default(): + host = FakeHostDecisionNode() + decision_id = _begin_pending(host) + pending = host._pending_action_error_decisions[decision_id] + pending["timer"].cancel() + pending["error_info"]["expires_at"] = time.time() - 1 + + assert not host.handle_action_error_decision( + decision_id, + "job-1", + _decision(decision_id, "retry"), + ) + + assert not host._pending_action_error_decisions + assert not host.sent_goals + assert host.finished[0][1] == "failed" + assert host.bridge.resolved_reports[0]["selected_action"] == "abort" + assert host.bridge.resolved_reports[0]["reason"] == "decision_timeout" + resolved = host.get_resolved_action_error_decision( + decision_id, + "job-1", + "device-1", + decision_target=ERROR_DECISION_TARGET_BACKEND, + ) + assert resolved == host.bridge.resolved_reports[0] + + +def test_resolved_decision_replay_does_not_execute_twice(): + host = FakeHostDecisionNode() + decision_id = _begin_pending(host) + assert host.handle_action_error_decision( + decision_id, + "job-1", + _decision(decision_id, "skip"), + ) + + replayed = host.replay_action_error_decision_resolution( + decision_id, + "job-1", + "device-1", + decision_target=ERROR_DECISION_TARGET_BACKEND, + ) + + assert replayed == host.bridge.resolved_reports[0] + assert host.bridge.resolved_reports == [replayed, replayed] + assert len(host.finished) == 1 + + def test_host_retry_limit_fails_closed(): host = FakeHostDecisionNode() host._error_execution_contexts["job-1"]["retry_count"] = 1 @@ -875,13 +1055,15 @@ def test_cancel_pending_error_decision_closes_timer_and_rejects_late_reply(): assert not timer.is_alive() assert not host._pending_action_error_decisions assert not host._error_execution_contexts - assert host.finished[0][1] == "failed" - assert host.bridge.resolved_reports[0]["selected_action"] == "cancel" + assert host.finished[0][1] == "canceled" + assert host.bridge.resolved_reports[0]["selected_action"] == "abort" assert host.bridge.resolved_reports[0]["reason"] == "job_canceled" + assert not host.cancel_job("job-1") + assert len(host.bridge.resolved_reports) == 1 assert not host.handle_action_error_decision( decision_id, "job-1", - {"action": "retry"}, + _decision(decision_id, "retry"), ) @@ -954,7 +1136,7 @@ def test_host_dispatches_registered_fallback_action(): assert host.handle_action_error_decision( decision_id, "job-1", - {"action": "reset_connection"}, + _decision(decision_id, "reset_connection"), ) args, kwargs = host.sent_goals[0] @@ -972,12 +1154,12 @@ def test_host_rejects_unconfigured_backend_option_without_consuming_pending(): assert not host.handle_action_error_decision( decision_id, "job-1", - {"action": "force_success"}, + _decision(decision_id, "force_success"), ) assert decision_id in host._pending_action_error_decisions assert host.handle_action_error_decision( decision_id, "job-1", - {"action": "abort"}, + _decision(decision_id, "abort"), ) diff --git a/unilabos/app/model.py b/unilabos/app/model.py index 448f4eae4..53417cc22 100644 --- a/unilabos/app/model.py +++ b/unilabos/app/model.py @@ -74,6 +74,9 @@ class JobAddReq(BaseModel): class ErrorDecisionIn(BaseModel): """Host 微后端提交的异常处理决策。""" + decision_id: str = Field(min_length=1) + job_id: str = Field(min_length=1) + device_id: str = Field(min_length=1) action: str = "" option: Any = None result: Any = None diff --git a/unilabos/app/web/api.py b/unilabos/app/web/api.py index e8ae71291..981088937 100644 --- a/unilabos/app/web/api.py +++ b/unilabos/app/web/api.py @@ -1334,8 +1334,18 @@ def api_submit_action_error_decision(decision_id: str, req: ErrorDecisionIn): decision = req.dict(exclude_unset=True) isok, data = submit_action_error_decision(decision_id, decision) if not isok: + error_code = str(data.get("error_code") or "") raise HTTPException( - status_code=404, + status_code=( + 409 + if error_code in { + "decision_expired", + "decision_identity_mismatch", + } + else 422 + if error_code == "decision_identity_required" + else 404 + ), detail=data.get("error", "Pending decision not found"), ) return data diff --git a/unilabos/app/web/controller.py b/unilabos/app/web/controller.py index 6f6d4136d..16192de44 100644 --- a/unilabos/app/web/controller.py +++ b/unilabos/app/web/controller.py @@ -106,6 +106,7 @@ def store_job_result( "success": 4, # SUCCEEDED "failed": 6, # ABORTED "cancelled": 5, # CANCELED + "canceled": 5, # CANCELED (canonical spelling) "running": 2, # EXECUTING } status_int = status_map.get(status, 0) @@ -374,12 +375,44 @@ def submit_action_error_decision( decision_id = str(decision_id or "") if not decision_id: return False, {"error": "decision_id is required"} + body_decision_id = str(decision.get("decision_id") or "") + job_id = str(decision.get("job_id") or "") + device_id = str(decision.get("device_id") or "") + if not body_decision_id or not job_id or not device_id: + return False, { + "error": "decision_id, job_id and device_id are required", + "error_code": "decision_identity_required", + } + if body_decision_id != decision_id: + return False, { + "error": "path decision_id does not match request body", + "error_code": "decision_identity_mismatch", + } if not host_node.handle_action_error_decision( decision_id, - "", + job_id, decision, decision_target=ERROR_DECISION_TARGET_MICRO_BACKEND, ): + resolved = host_node.get_resolved_action_error_decision( + decision_id, + job_id, + device_id, + decision_target=ERROR_DECISION_TARGET_MICRO_BACKEND, + ) + if resolved is not None: + if resolved.get("reason") == "decision_timeout": + return False, { + "error": "action error decision expired", + "error_code": "decision_expired", + "resolution": resolved, + } + return True, { + "decision_id": decision_id, + "status": "resolved", + "replayed": True, + "resolution": resolved, + } return False, {"error": "pending action error decision not found or mismatched"} return True, {"decision_id": decision_id, "status": "delivered"} diff --git a/unilabos/app/ws_client.py b/unilabos/app/ws_client.py index aa98ab647..172889713 100644 --- a/unilabos/app/ws_client.py +++ b/unilabos/app/ws_client.py @@ -763,11 +763,11 @@ async def _handle_job_error_decision(self, data: Dict[str, Any]): decision_id = str(data.get("decision_id") or "") job_id = str(data.get("job_id") or "") device_id = str(data.get("device_id") or "") - if not decision_id and not job_id: - logger.warning("[MessageProcessor] job_error_decision missing decision_id and job_id") - return - if not device_id: - logger.warning("[MessageProcessor] job_error_decision missing device_id") + if not decision_id or not job_id or not device_id: + logger.warning( + "[MessageProcessor] job_error_decision requires " + "decision_id, job_id and device_id" + ) return host_node = HostNode.get_instance(0) @@ -780,6 +780,18 @@ async def _handle_job_error_decision(self, data: Dict[str, Any]): dict(data), decision_target=ERROR_DECISION_TARGET_BACKEND, ): + replayed = host_node.replay_action_error_decision_resolution( + decision_id, + job_id, + device_id, + decision_target=ERROR_DECISION_TARGET_BACKEND, + ) + if replayed is not None: + logger.info( + f"[MessageProcessor] Replayed resolved error decision " + f"decision={decision_id} job={job_id[:8]}" + ) + return logger.warning( f"[MessageProcessor] No pending error decision matched " f"decision={decision_id} job={job_id[:8]} device={device_id}" @@ -1560,7 +1572,7 @@ def cache_job_start_response(self, item: QueueItem, message: Dict[str, Any], sta def replay_cached_job_start_response(self, job_id: str, task_id: str) -> bool: """回放同一 (task_id, job_id) 已缓存的最终结果。 - 仅当已缓存到 success/failed 的终态结果时才回放；若原任务仍在执行 + 仅当已缓存到 success/failed/canceled 的终态结果时才回放；若原任务仍在执行 (只缓存了 running 中间态)，返回 False，由调用方决定如何处理。 """ key = self._job_start_cache_key(job_id, task_id) @@ -1571,7 +1583,7 @@ def replay_cached_job_start_response(self, job_id: str, task_id: str) -> bool: cached = self._job_start_cache.get(key) if cached is None or cached.response_message is None: return False - if cached.response_status not in ("success", "failed"): + if cached.response_status not in ("success", "failed", "canceled"): return False message = copy.deepcopy(cached.response_message) status = cached.response_status @@ -1657,7 +1669,7 @@ def publish_job_status( job_log = format_job_log(item.job_id, item.task_id, item.device_id, item.action_name) # 拦截最终结果状态，与原版本逻辑一致 - if status in ["success", "failed"]: + if status in ["success", "failed", "canceled"]: self._job_running_last_sent.pop(item.job_id, None) host_node = HostNode.get_instance(0) @@ -1670,7 +1682,7 @@ def publish_job_status( self.queue_processor.handle_job_completed(item.job_id, status) cached_status = self.get_cached_job_start_response_status(item.job_id, item.task_id) - if cached_status in ["success", "failed"]: + if cached_status in ["success", "failed", "canceled"]: # 断线重连时，旧 READY 占位可能在结果已回放后触发 timeout failed。 # 已有终态时不允许重复终态覆盖缓存或再次发送，success 也不允许被 failed 降级。 if cached_status == "success" or cached_status == status: @@ -1746,6 +1758,23 @@ def publish_job_error_decision_resolved(self, report: Dict[str, Any]) -> bool: {"action": "job_error_decision_resolved", "data": report} ) + def report_action_error_decisions(self) -> None: + """连接/重连后重放 Host 持有的 pending 与短期终态。""" + + if self.is_disabled or not self.is_connected(): + return + host_node = HostNode.get_instance(0) + if host_node is None: + return + for report in host_node.get_pending_action_error_decisions( + ERROR_DECISION_TARGET_BACKEND, + ): + self.publish_job_error_decision_required(report) + for report in host_node.get_resolved_action_error_decisions( + ERROR_DECISION_TARGET_BACKEND, + ): + self.publish_job_error_decision_resolved(report) + def send_ping(self, ping_id: str, timestamp: float) -> None: """发送ping消息""" if self.is_disabled or not self.is_connected(): @@ -1897,6 +1926,7 @@ def publish_host_ready(self) -> None: # 先上报全量锁快照，再发 host_ready：借助发送队列 FIFO 顺序， # 服务端会先收到 report_action_lock、再收到 host_node_ready。 # 启动时全部 free，重连时按 DeviceActionManager 反映正在运行/排队的 busy，实现锁状态对齐。 + self.report_action_error_decisions() self.report_all_action_locks() self.message_processor.send_message(message) logger.info(f"[WebSocketClient] Host node ready signal published with {len(devices)} devices") diff --git a/unilabos/ros/nodes/presets/host_node.py b/unilabos/ros/nodes/presets/host_node.py index 563db1713..f0f17a2ab 100644 --- a/unilabos/ros/nodes/presets/host_node.py +++ b/unilabos/ros/nodes/presets/host_node.py @@ -155,6 +155,7 @@ class HostNode(BaseROS2DeviceNode): DeviceActionStatus ) _resource_tracker: ClassVar[DeviceNodeResourceTracker] = DeviceNodeResourceTracker() # 资源管理器实例 + _ACTION_ERROR_DECISION_TOMBSTONE_TTL_SECONDS: ClassVar[float] = 3600.0 @classmethod def get_instance(cls, timeout=None) -> Optional["HostNode"]: @@ -371,6 +372,7 @@ def __init__( # 异常决策只存在于 Host：设备返回结构化失败，Host 保留原 job 并负责恢复动作。 self._error_execution_contexts: Dict[str, Dict[str, Any]] = {} self._pending_action_error_decisions: Dict[str, Dict[str, Any]] = {} + self._resolved_action_error_decisions: Dict[str, Dict[str, Any]] = {} self._pending_action_error_decisions_lock = threading.RLock() # cancel 可能发生在 Goal 等待响应、执行中或等待异常决策三个阶段。 self._canceled_jobs: Set[str] = set() @@ -1351,15 +1353,32 @@ def get_pending_action_error_decisions( ] return reports - def _publish_action_error_decision_resolved( + def _prune_action_error_decision_tombstones_locked( + self, + now: Optional[float] = None, + ) -> None: + """清理 Host 内存中的短期决策终态；后端仍负责持久审计。""" + + current = time.time() if now is None else now + tombstones = getattr(self, "_resolved_action_error_decisions", {}) + stale_ids = [ + decision_id + for decision_id, tombstone in tombstones.items() + if float(tombstone.get("retain_until", 0.0)) <= current + ] + for decision_id in stale_ids: + tombstones.pop(decision_id, None) + + def _remember_action_error_decision_resolution_locked( self, pending: Dict[str, Any], selected_action: str, - reason: str = "", + reason: str, ) -> Dict[str, Any]: - """发布决策终态审计；观测/通信失败不影响实际决策。""" + """在消费 pending 的同一临界区记录可重放终态。""" item = pending["item"] + resolved_at = time.time() resolved_report = { "decision_id": pending["decision_id"], "job_id": pending["job_id"], @@ -1368,8 +1387,135 @@ def _publish_action_error_decision_resolved( "action_name": item.action_name, "selected_action": selected_action, "reason": reason, - "resolved_at": time.time(), + "resolved_at": resolved_at, } + tombstones = getattr(self, "_resolved_action_error_decisions", None) + if tombstones is None: + tombstones = {} + self._resolved_action_error_decisions = tombstones + self._prune_action_error_decision_tombstones_locked(resolved_at) + tombstones[pending["decision_id"]] = { + "report": deepcopy(resolved_report), + "decision_target": pending.get("decision_target"), + "retain_until": ( + resolved_at + self._ACTION_ERROR_DECISION_TOMBSTONE_TTL_SECONDS + ), + } + return resolved_report + + def get_resolved_action_error_decision( + self, + decision_id: str, + job_id: str, + device_id: str, + *, + decision_target: Optional[str] = None, + ) -> Optional[Dict[str, Any]]: + """按完整身份读取短期终态，供断线重放和 REST 幂等响应。""" + + if not decision_id or not job_id or not device_id: + return None + with self._pending_action_error_decisions_lock: + self._prune_action_error_decision_tombstones_locked() + tombstone = getattr( + self, + "_resolved_action_error_decisions", + {}, + ).get(decision_id) + if tombstone is None: + return None + if ( + decision_target is not None + and tombstone.get("decision_target") != decision_target + ): + return None + report = tombstone.get("report") + if not isinstance(report, dict): + return None + if ( + report.get("job_id") != job_id + or report.get("device_id") != device_id + ): + return None + return deepcopy(report) + + def get_resolved_action_error_decisions( + self, + decision_target: Optional[str] = None, + ) -> List[Dict[str, Any]]: + """读取仍在 Host tombstone 窗口内的决策终态。""" + + with self._pending_action_error_decisions_lock: + self._prune_action_error_decision_tombstones_locked() + return [ + deepcopy(tombstone["report"]) + for tombstone in getattr( + self, + "_resolved_action_error_decisions", + {}, + ).values() + if isinstance(tombstone.get("report"), dict) + and ( + decision_target is None + or tombstone.get("decision_target") == decision_target + ) + ] + + def replay_action_error_decision_resolution( + self, + decision_id: str, + job_id: str, + device_id: str, + *, + decision_target: Optional[str] = None, + ) -> Optional[Dict[str, Any]]: + """向原决策端重放已解决终态，不再次执行恢复动作。""" + + resolved_report = self.get_resolved_action_error_decision( + decision_id, + job_id, + device_id, + decision_target=decision_target, + ) + if resolved_report is None: + return None + if decision_target == ERROR_DECISION_TARGET_BACKEND: + for bridge in self.bridges: + publish_resolved = getattr( + bridge, + "publish_job_error_decision_resolved", + None, + ) + if not callable(publish_resolved): + continue + try: + if publish_resolved(deepcopy(resolved_report)): + break + except Exception as ex: # noqa: BLE001 - 重放失败不改变终态 + self.lab_logger().warning( + f"[Host Node] 异常决策终态重放失败: {bridge!r}: {ex}" + ) + return resolved_report + + def _publish_action_error_decision_resolved( + self, + pending: Dict[str, Any], + selected_action: str, + reason: str = "", + resolved_report: Optional[Dict[str, Any]] = None, + ) -> Dict[str, Any]: + """发布决策终态审计；观测/通信失败不影响实际决策。""" + + item = pending["item"] + if resolved_report is None: + with self._pending_action_error_decisions_lock: + resolved_report = ( + self._remember_action_error_decision_resolution_locked( + pending, + selected_action, + reason, + ) + ) self._emit_local_action_event( item, "job_error_decision_resolved", @@ -1400,6 +1546,7 @@ def _handle_action_error_decision_timeout(self, decision_id: str) -> None: return error_info = pending["error_info"] job_id = pending["job_id"] + device_id = pending["item"].device_id timeout_action = str( error_info.get("default_on_decision_timeout", "abort") ) @@ -1415,9 +1562,11 @@ def _handle_action_error_decision_timeout(self, decision_id: str) -> None: { "decision_id": decision_id, "job_id": job_id, + "device_id": device_id, "action": timeout_action, "reason": "decision_timeout", }, + _timeout_resolution=True, ) def handle_action_error_decision( @@ -1427,18 +1576,27 @@ def handle_action_error_decision( decision: Dict[str, Any], *, decision_target: Optional[str] = None, + _timeout_resolution: bool = False, ) -> bool: """在 Host 上处理决策，并通过现有 ActionClient 发起恢复动作。""" + body_decision_id = str(decision.get("decision_id") or "") + body_job_id = str(decision.get("job_id") or "") + body_device_id = str(decision.get("device_id") or "") + if ( + not decision_id + or not job_id + or not body_decision_id + or not body_job_id + or not body_device_id + or body_decision_id != decision_id + or body_job_id != job_id + ): + return False + + caller_accepted = True with self._pending_action_error_decisions_lock: - pending = self._pending_action_error_decisions.get(decision_id) if decision_id else None - if pending is None and job_id: - matches = [ - candidate - for candidate in self._pending_action_error_decisions.values() - if candidate.get("job_id") == job_id - ] - pending = matches[0] if len(matches) == 1 else None + pending = self._pending_action_error_decisions.get(decision_id) if ( pending is None or pending.get("resolving") @@ -1450,18 +1608,38 @@ def handle_action_error_decision( and pending.get("decision_target") != decision_target ): return False - if job_id and pending["job_id"] != job_id: + if pending["job_id"] != job_id: return False - body_decision_id = str(decision.get("decision_id") or "") - body_job_id = str(decision.get("job_id") or "") - if body_decision_id and body_decision_id != pending["decision_id"]: - return False - if body_job_id and body_job_id != pending["job_id"]: - return False - body_device_id = str(decision.get("device_id") or "") - if body_device_id and body_device_id != pending["item"].device_id: + if body_device_id != pending["item"].device_id: return False + expires_at = float(pending["error_info"].get("expires_at", 0.0)) + if ( + not _timeout_resolution + and expires_at > 0.0 + and time.time() >= expires_at + ): + timeout_action = str( + pending["error_info"].get( + "default_on_decision_timeout", + "abort", + ) + ) + allowed_actions = { + str(option.get("action")) + for option in pending["error_info"].get("options", []) + } + if timeout_action != "abort" and timeout_action not in allowed_actions: + timeout_action = "abort" + decision = { + "decision_id": decision_id, + "job_id": job_id, + "device_id": body_device_id, + "action": timeout_action, + "reason": "decision_timeout", + } + caller_accepted = False + selected_option = decision.get("option") if isinstance(selected_option, dict): selected = str(selected_option.get("action") or "abort") @@ -1486,12 +1664,18 @@ def handle_action_error_decision( timer = pending.get("timer") if timer is not None: timer.cancel() + resolved_report = self._remember_action_error_decision_resolution_locked( + pending, + selected, + str(decision.get("reason") or ""), + ) item = pending["item"] self._publish_action_error_decision_resolved( pending, selected, str(decision.get("reason") or ""), + resolved_report, ) if selected == "abort": self._finish_error_handled_job( @@ -1500,7 +1684,7 @@ def handle_action_error_decision( pending["return_info"], pending["result_data"], ) - return True + return caller_accepted if selected == "skip": return_value = decision.get("result", decision.get("return_value")) @@ -1513,7 +1697,7 @@ def handle_action_error_decision( result_data = dict(pending["result_data"]) result_data["return_info"] = json.dumps(return_info, ensure_ascii=False) self._finish_error_handled_job(item, "success", return_info, result_data) - return True + return caller_accepted execution_context = pending.get("execution_context") if selected == "retry": @@ -1524,7 +1708,7 @@ def handle_action_error_decision( serialize_result_info("缺少原动作上下文，无法重试", False, {}), pending["result_data"], ) - return True + return caller_accepted retries = int(execution_context.get("retry_count", 0)) max_retries = int(pending["error_info"].get("max_retries", 3)) if retries >= max_retries: @@ -1538,7 +1722,7 @@ def handle_action_error_decision( ), pending["result_data"], ) - return True + return caller_accepted execution_context["retry_count"] = retries + 1 try: self.send_goal( @@ -1557,7 +1741,7 @@ def handle_action_error_decision( serialize_result_info(traceback.format_exc(), False, {}), {"error": str(ex)}, ) - return True + return caller_accepted fallback = option.get("fallback_action") if isinstance(option, dict) else None if not isinstance(fallback, dict): @@ -1570,7 +1754,7 @@ def handle_action_error_decision( suc_type=SUCCESS_TYPE_OPERATOR_INTERVENTION, ) self._finish_error_handled_job(item, "success", return_info, {}) - return True + return caller_accepted self._finish_error_handled_job( item, "failed", @@ -1581,7 +1765,7 @@ def handle_action_error_decision( ), pending["result_data"], ) - return True + return caller_accepted fallback_name = str(fallback.get("action_name") or "") action_mappings = self._action_value_mappings.get(item.device_id, {}) @@ -1601,7 +1785,7 @@ def handle_action_error_decision( ), pending["result_data"], ) - return True + return caller_accepted fallback_item = type(item)( task_type=item.task_type, @@ -1632,7 +1816,7 @@ def handle_action_error_decision( serialize_result_info(traceback.format_exc(), False, {}), {"error": str(ex)}, ) - return True + return caller_accepted def goal_response_callback( self, @@ -1820,6 +2004,7 @@ def cancel_job(self, job_id: str) -> bool: """取消运行中、等待 Goal 响应或等待异常决策的逻辑 job。""" pending = None + resolved_report = None with self._pending_action_error_decisions_lock: for decision_id, candidate in list( self._pending_action_error_decisions.items() @@ -1831,6 +2016,13 @@ def cancel_job(self, job_id: str) -> bool: timer = pending.get("timer") if timer is not None: timer.cancel() + resolved_report = ( + self._remember_action_error_decision_resolution_locked( + pending, + "abort", + "job_canceled", + ) + ) break goal_handle = self._goals.get(job_id) @@ -1846,12 +2038,13 @@ def cancel_job(self, job_id: str) -> bool: item = pending["item"] self._publish_action_error_decision_resolved( pending, - "cancel", + "abort", "job_canceled", + resolved_report, ) self._finish_error_handled_job( item, - "failed", + "canceled", serialize_result_info("Job was cancelled", False, {}), {}, )