From 10da24fc8e36b1d86102b1b05bd1c4c23eb963e7 Mon Sep 17 00:00:00 2001 From: Joseph Ibrahim Date: Thu, 2 Jul 2026 10:24:22 -0400 Subject: [PATCH 1/4] fix: make 11 lying/broken MCP tools honest (M1 stop-the-bleeding) Every fix traces to the hand-verified 2026-06-11 codebase review: - codegen delete_actor/set_transform: resolve level actors via load_object (asset API returned None for subobject paths -> NOT_FOUND) - blueprints set_component_property: JS `true` -> True (NameError after the mutation already ran); spawn_blueprint label indent (8 spaces in the nested template -> was IndentationError, never spawned with label) - level load_level: report the actual load_level() return instead of unconditional loaded:true on a destructive op - editor focus_actor: success only when the focus path actually ran; drop duplicate deprecated selection call - scene get_actor_details: actor.hidden attribute (is_hidden() is not a UFUNCTION -> AttributeError) - assets find_assets: escape_for_fstring on the pattern (only codegen path that skipped escaping; broke on quotes/backslashes) - mograph create_cloner: layout/count/spacing/mesh were validated then DISCARDED - now configured post-spawn via _safe_set with per-property applied/skipped reporting (lighting.py pattern) + mesh child attach - perception fallback: take_high_res_screenshot completes on a later frame - split into trigger -> poll -> read editor round-trips with honest capture_status (was: image:"" with success:true) - metrics: uptime rounded to 3 decimals (round(x,1) + coarse Py3.12 monotonic made test_uptime flake) - mcp_server: import remote_control directly; drop sys.path hack and the E402 exemption -> module imports cleanly outside the repo root 415 tests green; ruff clean. Exec-sim regression harness lands separately. Co-Authored-By: Claude Fable 5 --- pyproject.toml | 4 -- remote_control/codegen.py | 6 ++- ue_mcp/mcp_server.py | 8 +-- ue_mcp/metrics.py | 2 +- ue_mcp/tools/assets.py | 2 +- ue_mcp/tools/blueprints.py | 6 ++- ue_mcp/tools/editor.py | 11 ++-- ue_mcp/tools/level.py | 5 +- ue_mcp/tools/mograph.py | 77 +++++++++++++++++++++++---- ue_mcp/tools/perception.py | 106 ++++++++++++++++++++++++------------- ue_mcp/tools/scene.py | 2 +- 11 files changed, 159 insertions(+), 70 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 6a554c0..109b374 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,7 +56,3 @@ line-length = 120 select = ["E", "F", "I", "N", "UP", "B"] ignore = ["E501"] -[tool.ruff.lint.per-file-ignores] -# mcp_server.py inserts the repo root on sys.path before importing first-party -# modules, so those imports legitimately come after executable statements. -"ue_mcp/mcp_server.py" = ["E402"] diff --git a/remote_control/codegen.py b/remote_control/codegen.py index a25a8d7..380be7b 100644 --- a/remote_control/codegen.py +++ b/remote_control/codegen.py @@ -36,10 +36,12 @@ def spawn_actor_code( @staticmethod def delete_actor_code(actor_path: str) -> str: + # Level actors are subobjects (…:PersistentLevel.Name) — the Content-Browser + # asset API returns None for them; load_object resolves both forms. return f""" import unreal subsystem = unreal.get_editor_subsystem(unreal.EditorActorSubsystem) -actor = unreal.EditorAssetLibrary.load_asset("{actor_path}") +actor = unreal.load_object(None, "{actor_path}") if actor: subsystem.destroy_actor(actor) print("RESULT:DELETED") @@ -77,7 +79,7 @@ def set_actor_transform_code( scale: tuple[float, float, float] | None, ) -> str: lines = ["import unreal"] - lines.append(f'actor = unreal.EditorAssetLibrary.load_asset("{actor_path}")') + lines.append(f'actor = unreal.load_object(None, "{actor_path}")') lines.append("if actor:") if location: lines.append(f" actor.set_actor_location(unreal.Vector({location[0]}, {location[1]}, {location[2]}), False, False)") diff --git a/ue_mcp/mcp_server.py b/ue_mcp/mcp_server.py index 3d1a11d..b938554 100644 --- a/ue_mcp/mcp_server.py +++ b/ue_mcp/mcp_server.py @@ -14,17 +14,11 @@ import glob import json import os -import sys import tempfile -# Ensure the ue-bridge root is importable (for remote_control_bridge) -_parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -if _parent_dir not in sys.path: - sys.path.insert(0, _parent_dir) - from mcp.server.fastmcp import FastMCP -from remote_control_bridge import BASE_URL, AsyncUnrealRemoteControl +from remote_control import BASE_URL, AsyncUnrealRemoteControl from ue_mcp.__version__ import __version__ from ue_mcp.metrics import metrics from ue_mcp.tools import register_all_tools diff --git a/ue_mcp/metrics.py b/ue_mcp/metrics.py index 71795a2..e292bf6 100644 --- a/ue_mcp/metrics.py +++ b/ue_mcp/metrics.py @@ -51,7 +51,7 @@ def snapshot(self) -> dict[str, Any]: "p95_ms": round(sorted_s[int(len(sorted_s) * 0.95)] * 1000, 1), } return { - "uptime_s": round(uptime, 1), + "uptime_s": round(uptime, 3), "counters": dict(self._counters), "latencies": latency_stats, } diff --git a/ue_mcp/tools/assets.py b/ue_mcp/tools/assets.py index 1124236..08a079e 100644 --- a/ue_mcp/tools/assets.py +++ b/ue_mcp/tools/assets.py @@ -35,7 +35,7 @@ async def find_assets(search_pattern: str, class_filter: str | None = None) -> s if err := sanitize_class_name(class_filter, "class_filter"): return make_error(err) - result = await ue.find_assets(search_pattern, class_filter=class_filter) + result = await ue.find_assets(escape_for_fstring(search_pattern), class_filter=class_filter) return json.dumps(result, indent=2) @server.tool( diff --git a/ue_mcp/tools/blueprints.py b/ue_mcp/tools/blueprints.py index bfa1691..d6e8049 100644 --- a/ue_mcp/tools/blueprints.py +++ b/ue_mcp/tools/blueprints.py @@ -205,7 +205,7 @@ async def set_component_property( val = asset try: comp.set_editor_property("{safe_prop}", val) - print("RESULT:" + json.dumps({{"set": true, "component": "{safe_cc}", "property": "{safe_prop}"}})) + print("RESULT:" + json.dumps({{"set": True, "component": "{safe_cc}", "property": "{safe_prop}"}})) except Exception as e: print("RESULT:" + json.dumps({{"error": str(e)}})) """ @@ -373,7 +373,9 @@ async def spawn_blueprint( label_line = "" if label: safe_lbl = escape_for_fstring(label) - label_line = f'\n actor.set_actor_label("{safe_lbl}")' + # This template's `if actor:` is nested one level deep — the label + # statement must sit at 8 spaces, unlike codegen.spawn_actor_code. + label_line = f'\n actor.set_actor_label("{safe_lbl}")' code = f""" import unreal, json diff --git a/ue_mcp/tools/editor.py b/ue_mcp/tools/editor.py index 78af21a..ca8157a 100644 --- a/ue_mcp/tools/editor.py +++ b/ue_mcp/tools/editor.py @@ -153,16 +153,17 @@ async def focus_actor(actor_label: str) -> str: if actor is None: print("RESULT:" + json.dumps({{"error": "Actor not found: {safe_label}"}})) else: - # Select the actor and focus subsystem.set_selected_level_actors([actor]) - # Use editor utility to focus on selection - unreal.EditorLevelLibrary.set_selected_level_actors([actor]) - # Trigger viewport focus + focused = False if hasattr(unreal, 'LevelEditorSubsystem'): le_sub = unreal.get_editor_subsystem(unreal.LevelEditorSubsystem) if hasattr(le_sub, 'focus_on_selected_actors'): le_sub.focus_on_selected_actors() - print("RESULT:" + json.dumps({{"focused": "{safe_label}"}})) + focused = True + if focused: + print("RESULT:" + json.dumps({{"focused": "{safe_label}"}})) + else: + print("RESULT:" + json.dumps({{"error": "No viewport focus method available", "selected": "{safe_label}"}})) """ result = await ue.execute_python(code) return json.dumps(result, indent=2) diff --git a/ue_mcp/tools/level.py b/ue_mcp/tools/level.py index 922cf8b..af31441 100644 --- a/ue_mcp/tools/level.py +++ b/ue_mcp/tools/level.py @@ -60,7 +60,10 @@ async def load_level(level_path: str) -> str: try: success = unreal.EditorLevelLibrary.load_level("{safe_path}") - print("RESULT:" + json.dumps({{"loaded": True, "level": "{safe_path}"}})) + if success: + print("RESULT:" + json.dumps({{"loaded": True, "level": "{safe_path}"}})) + else: + print("RESULT:" + json.dumps({{"loaded": False, "error": "load_level returned False", "level": "{safe_path}"}})) except Exception as e: print("RESULT:" + json.dumps({{"error": str(e)}})) """ diff --git a/ue_mcp/tools/mograph.py b/ue_mcp/tools/mograph.py index 40ee1da..48d91fd 100644 --- a/ue_mcp/tools/mograph.py +++ b/ue_mcp/tools/mograph.py @@ -51,30 +51,87 @@ async def create_cloner( return make_error(err) label_str = escape_for_fstring(label or "ClaudeCloner") + safe_mesh = escape_for_fstring(mesh_path) + # ClonerEffector clones its ATTACHED child actors; layout/count/spacing + # live on the cloner component and its active layout object. Property + # names could not be verified against a live editor, so every write goes + # through _safe_set and is reported applied/skipped (lighting.py pattern) + # instead of silently pretending. code = f""" -import unreal +import unreal, json subsystem = unreal.get_editor_subsystem(unreal.EditorActorSubsystem) -# Spawn the Cloner actor cloner_class = unreal.find_class("ClonerActor") or unreal.find_class("ACEClonerActor") if cloner_class is None: - # Try loading from the plugin module cloner_class = unreal.load_class(None, "/Script/ClonerEffector.ClonerActor") -if cloner_class: +if cloner_class is None: + print("RESULT:" + json.dumps({{"error": "CLASS_NOT_FOUND - ClonerEffector plugin may not be loaded"}})) +else: cloner = subsystem.spawn_actor_from_class( cloner_class, unreal.Vector({x}, {y}, {z}), unreal.Rotator(0, 0, 0) ) - if cloner: - cloner.set_actor_label("{label_str}") - print("RESULT:CREATED " + cloner.get_path_name()) + if cloner is None: + print("RESULT:" + json.dumps({{"error": "SPAWN_FAILED"}})) else: - print("RESULT:SPAWN_FAILED") -else: - print("RESULT:CLASS_NOT_FOUND - ClonerEffector plugin may not be loaded") + cloner.set_actor_label("{label_str}") + applied, skipped = [], [] + + def _safe_set(obj, name, value): + try: + obj.set_editor_property(name, value) + applied.append(name) + except Exception as e: + skipped.append(name + ": " + str(e)[:80]) + + comp = None + comp_class = getattr(unreal, "CEClonerComponent", None) + if comp_class: + comp = cloner.get_component_by_class(comp_class) + if comp is None: + skipped.append("layout/count/spacing: CEClonerComponent not found on actor") + else: + _safe_set(comp, "layout_name", "{layout}") + layout_obj = None + try: + layout_obj = comp.get_editor_property("active_layout") + except Exception: + pass + target = layout_obj if layout_obj is not None else comp + _safe_set(target, "count_x", {count_x}) + _safe_set(target, "count_y", {count_y}) + _safe_set(target, "count_z", {count_z}) + _safe_set(target, "spacing_x", {spacing}) + _safe_set(target, "spacing_y", {spacing}) + _safe_set(target, "spacing_z", {spacing}) + + mesh_attached = False + try: + mesh = unreal.EditorAssetLibrary.load_asset("{safe_mesh}") + if mesh: + child = subsystem.spawn_actor_from_class( + unreal.StaticMeshActor, unreal.Vector({x}, {y}, {z}), unreal.Rotator(0, 0, 0)) + if child: + child.static_mesh_component.set_static_mesh(mesh) + child.attach_to_actor(cloner, "", unreal.AttachmentRule.KEEP_RELATIVE, + unreal.AttachmentRule.KEEP_RELATIVE, + unreal.AttachmentRule.KEEP_RELATIVE, False) + mesh_attached = True + else: + skipped.append("mesh: asset not found {safe_mesh}") + except Exception as e: + skipped.append("mesh: " + str(e)[:80]) + + print("RESULT:" + json.dumps({{ + "created": cloner.get_path_name(), + "layout": "{layout}", + "applied": applied, + "skipped": skipped, + "mesh_attached": mesh_attached, + }})) """ result = await ue.execute_python(code) return json.dumps(result, indent=2) diff --git a/ue_mcp/tools/perception.py b/ue_mcp/tools/perception.py index 44850a7..81d82f1 100644 --- a/ue_mcp/tools/perception.py +++ b/ue_mcp/tools/perception.py @@ -9,6 +9,7 @@ from __future__ import annotations +import asyncio import json import logging import os @@ -24,6 +25,10 @@ PERCEPTION_URL = os.environ.get("UE_PERCEPTION_URL", "http://localhost:30011") PERCEPTION_TIMEOUT = 5.0 +# take_high_res_screenshot completes on a later frame — the fallback polls for +# the file across separate editor round-trips (see _fallback_capture). +FALLBACK_POLL_ATTEMPTS = 10 +FALLBACK_POLL_INTERVAL_S = 0.5 BRIDGE_DIR = Path.home() / ".translators" @@ -92,67 +97,49 @@ async def _perception_request(method: str, path: str, body: dict | None = None) async def _fallback_capture(ue, width: int, height: int, format: str) -> dict: - """Fallback: capture via SceneCapture2D + Python in the editor. + """Fallback: capture via editor screenshot + Python in the editor. - This re-renders the scene (performance cost) but works without the C++ plugin. + take_high_res_screenshot completes on a LATER frame, so the trigger and the + file read must be separate editor executions — one combined exec always saw + a missing file and (before the fix) reported success with an empty image. """ - code = f""" -import unreal, json, base64, os, tempfile + trigger_code = f""" +import unreal, json, tempfile -# Get viewport info world = unreal.EditorLevelLibrary.get_editor_world() subsystem = unreal.get_editor_subsystem(unreal.EditorActorSubsystem) actors = subsystem.get_all_level_actors() level_name = world.get_name() if world else "Unknown" -# Get active viewport camera ecs = unreal.get_editor_subsystem(unreal.UnrealEditorSubsystem) loc, rot = unreal.Vector(), unreal.Rotator() try: - vp = unreal.EditorLevelLibrary - loc = ecs.get_level_viewport_camera_info()[0] if hasattr(ecs, 'get_level_viewport_camera_info') else unreal.Vector() - rot = ecs.get_level_viewport_camera_info()[1] if hasattr(ecs, 'get_level_viewport_camera_info') else unreal.Rotator() + if hasattr(ecs, 'get_level_viewport_camera_info'): + loc, rot = ecs.get_level_viewport_camera_info() except Exception: pass -# Selected actors selected = [] -sel = unreal.EditorUtilityLibrary.get_selected_assets() if hasattr(unreal, 'EditorUtilityLibrary') else [] try: - sel_actors = unreal.get_editor_subsystem(unreal.EditorActorSubsystem).get_selected_level_actors() - selected = [a.get_actor_label() for a in sel_actors] + selected = [a.get_actor_label() for a in subsystem.get_selected_level_actors()] except Exception: pass -# Capture via screenshot tmp_dir = tempfile.gettempdir().replace("\\\\", "/") out_path = tmp_dir + "/ue_perception_capture.{format}" -# Use high-res screenshot -success = False +trigger = "none" try: unreal.AutomationLibrary.take_high_res_screenshot({width}, {height}, out_path) - success = True + trigger = "automation" except Exception: - pass - -if not success: - # Fallback: use viewport screenshot command try: - cmd = f"HighResShot {width}x{height}" - unreal.SystemLibrary.execute_console_command(world, cmd) + unreal.SystemLibrary.execute_console_command(world, "HighResShot {width}x{height}") + trigger = "console" except Exception: pass -# Read and encode the image if it exists -image_b64 = "" -if os.path.exists(out_path): - with open(out_path, "rb") as f: - image_b64 = base64.b64encode(f.read()).decode("ascii") - os.remove(out_path) - -result = {{ - "image": image_b64, +print("RESULT:" + json.dumps({{ "width": {width}, "height": {height}, "format": "{format}", @@ -176,11 +163,58 @@ async def _fallback_capture(ue, width: int, height: int, format: str) -> dict: "delta_time": 0, "fps": 0 }}, - "fallback": True -}} -print("RESULT:" + json.dumps(result)) + "fallback": True, + "trigger": trigger, + "out_path": out_path +}})) +""" + triggered = await ue.execute_python(trigger_code) + meta = triggered.get("result") + if triggered.get("error") or not isinstance(meta, dict): + return triggered + + out_path = meta.pop("out_path", "") + meta["image"] = "" + + if meta.get("trigger") != "automation" or not out_path: + # The console-command route writes to the editor's own screenshot dir — + # we cannot poll for it, so report metadata-only honestly. + meta["capture_status"] = "untracked_trigger" if meta.get("trigger") == "console" else "trigger_failed" + return {"output": "", "error": None, "result": meta} + + poll_code = ( + "import os, json\n" + f'print("RESULT:" + json.dumps({{"exists": os.path.exists("{out_path}")}}))' + ) + found = False + for _ in range(FALLBACK_POLL_ATTEMPTS): + await asyncio.sleep(FALLBACK_POLL_INTERVAL_S) + chk = await ue.execute_python(poll_code) + chk_result = chk.get("result") + if isinstance(chk_result, dict) and chk_result.get("exists"): + found = True + break + + if not found: + meta["capture_status"] = "timeout" + meta["image_pending_path"] = out_path + return {"output": "", "error": None, "result": meta} + + read_code = f""" +import json, base64, os +with open("{out_path}", "rb") as f: + image_b64 = base64.b64encode(f.read()).decode("ascii") +os.remove("{out_path}") +print("RESULT:" + json.dumps({{"image": image_b64}})) """ - return await ue.execute_python(code) + read = await ue.execute_python(read_code) + read_result = read.get("result") + if isinstance(read_result, dict) and read_result.get("image"): + meta["image"] = read_result["image"] + meta["capture_status"] = "ok" + else: + meta["capture_status"] = "read_failed" + return {"output": "", "error": None, "result": meta} def _compute_scene_diff(snap1: dict, snap2: dict) -> dict: diff --git a/ue_mcp/tools/scene.py b/ue_mcp/tools/scene.py index 6bc2c9b..2fc992c 100644 --- a/ue_mcp/tools/scene.py +++ b/ue_mcp/tools/scene.py @@ -73,7 +73,7 @@ async def get_actor_details(actor_label: str) -> str: "location": {{"x": loc.x, "y": loc.y, "z": loc.z}}, "rotation": {{"pitch": rot.pitch, "yaw": rot.yaw, "roll": rot.roll}}, "scale": {{"x": scale.x, "y": scale.y, "z": scale.z}}, - "visible": actor.is_hidden() is False, + "visible": actor.hidden is False, "components": comp_list, "tags": tags, "parent": parent_label, From 454aa3f39e26200900590647ca7f15945bb737ad Mon Sep 17 00:00:00 2001 From: Joseph Ibrahim Date: Thu, 2 Jul 2026 10:50:20 -0400 Subject: [PATCH 2/4] test: exec-simulating codegen harness + honest undo/redo + materials param fix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit tests/exec_sim/ — the layer the mock suite could never be: a strict fake `unreal` module (phantom APIs deliberately absent), a registry classifying all 56 tools (49 CODEGEN / 7 DIRECT) with sentinel kwargs, and four gates per codegen tool: registry completeness, compile, exec-under-stub, and sentinel survival — plus scripted-failure honesty contracts for every historical lying-tool bug. Verified red on the unfixed tree (22 failures, all mapping 1:1 to known bugs), green after the fixes. Rides along: - ue_undo/ue_redo: honest not-implemented responses (no phantom-API probing, no editor round-trip); no editor-transaction route exists in the UE Python API and Epic's 5.8 MCP ships none either - ue_get_material_parameters: branch MaterialInstanceConstant vs base Material API families; str(name) keys (unreal.Name broke json.dumps) - old TestUndo/RedoAsync updated to the not-implemented contract 562 passed, 6 skipped; ruff clean. Co-Authored-By: Claude Fable 5 --- tests/exec_sim/README.md | 59 ++ tests/exec_sim/__init__.py | 1 + tests/exec_sim/conftest.py | 98 ++++ tests/exec_sim/registry.py | 381 +++++++++++++ tests/exec_sim/test_codegen_exec.py | 141 +++++ tests/exec_sim/test_honesty.py | 266 +++++++++ tests/exec_sim/unreal_stub.py | 809 ++++++++++++++++++++++++++++ tests/test_editor.py | 63 +-- ue_mcp/tools/editor.py | 76 +-- ue_mcp/tools/materials.py | 70 +-- 10 files changed, 1822 insertions(+), 142 deletions(-) create mode 100644 tests/exec_sim/README.md create mode 100644 tests/exec_sim/__init__.py create mode 100644 tests/exec_sim/conftest.py create mode 100644 tests/exec_sim/registry.py create mode 100644 tests/exec_sim/test_codegen_exec.py create mode 100644 tests/exec_sim/test_honesty.py create mode 100644 tests/exec_sim/unreal_stub.py diff --git a/tests/exec_sim/README.md b/tests/exec_sim/README.md new file mode 100644 index 0000000..59757db --- /dev/null +++ b/tests/exec_sim/README.md @@ -0,0 +1,59 @@ +# exec_sim — exec-simulating codegen harness + +The mock suite asserts on generated code *strings*; this harness **runs** the +generated UE editor Python against a strict fake `unreal` module and asserts on +*behavior*. It exists to catch the bug classes string-assertions are blind to: +syntax errors, NameErrors in success branches, args validated-then-discarded, +phantom (nonexistent) `unreal` APIs, and hard-coded success prints. + +## Files + +| File | Role | +|------|------| +| `unreal_stub.py` | `make_unreal_stub()` builds a fresh fake `unreal` module (strict: unknown top-level symbols raise `AttributeError`). `exec_generated()` installs it in `sys.modules` and runs a script, capturing stdout. `parse_result()` reuses the product RESULT-line parser. | +| `registry.py` | One `ToolEntry` per registered tool: canonical sentinel kwargs, `mode` (`CODEGEN`/`DIRECT`), `sentinel_checkable` kwargs, `expect_error`, notes. | +| `conftest.py` | Registers all tools once per session against a recording fake server and a `CaptureUE` (real `AsyncUnrealRemoteControl` delegation, `execute_python` captures instead of sending). | +| `test_codegen_exec.py` | The four generic gates (below), parametrized over every CODEGEN tool. | +| `test_honesty.py` | Scripted-failure contracts: force a real-world failure, assert the code does not claim success. | + +## The gates and what each catches + +1. **Registry completeness** — `REGISTRY` keys must equal the registered tool + names exactly. A new tool cannot dodge the harness. +2. **Compile** — generated Python must `compile()`. Catches interpolation / + indentation breakage (e.g. a label kwarg producing an `IndentationError`). +3. **Exec + honest success** — run under the default all-success stub: must + finish, print a `RESULT:` line, and not report failure. Catches phantom + APIs (`AttributeError`), bare JSON literals (`true` → `NameError`), + unserializable results (`unreal.Name` dict keys → `TypeError`), and dead + tools that can *never* succeed. +4. **Sentinel** — each `sentinel_checkable` kwarg's value must appear literally + (raw or `escape_for_fstring`-escaped) in the source. Catches + validated-then-discarded arguments. +5. **Honesty (scripted failure)** — `stub.configure(load_level=False)`-style + overrides force failure paths; the RESULT must not claim success. + +## Adding a new tool + +1. Register it as usual in `ue_mcp/tools/`. +2. Add a `ToolEntry` in `registry.py` (gate 1 fails until you do): + - `mode=CODEGEN` with distinctive sentinel kwargs (dyadic floats like + `433.25` survive `str()`/`json.dumps` exactly), listing in + `sentinel_checkable` every kwarg embedded *literally* in the script; + - or `mode=DIRECT` with a `notes` reason (HTTP-only, pass-through, pure + server-side...). +3. If the generated code uses a new `unreal` symbol, add it to the stub — + **only after verifying it exists in the real UE Python API**. Phantom + symbols staying absent is the whole point (`editor_undo`, + `transaction_undo`, and actor `is_hidden()` are deliberately missing). +4. If the tool needs a forced-failure contract, add a `stub.configure(...)` + flag and a test in `test_honesty.py`. + +## Notes + +- The stub's seeded world (actors `SENTINEL_LBL_9Q`, `SENTINEL_LBL_B2`, + `Cube_1`) matches the registry sentinels so success branches actually run. +- `ue_viewport_percept` is DIRECT (HTTP primary path), but its Python fallback + is exec-simmed via `perception._fallback_capture` in `test_honesty`. +- `ue_status` / `ue_health_check` live in `mcp_server.py`, outside + `register_all_tools`, and are out of scope here. diff --git a/tests/exec_sim/__init__.py b/tests/exec_sim/__init__.py new file mode 100644 index 0000000..88cd6dd --- /dev/null +++ b/tests/exec_sim/__init__.py @@ -0,0 +1 @@ +"""Exec-simulating codegen test harness (see README.md in this directory).""" diff --git a/tests/exec_sim/conftest.py b/tests/exec_sim/conftest.py new file mode 100644 index 0000000..bd9ee48 --- /dev/null +++ b/tests/exec_sim/conftest.py @@ -0,0 +1,98 @@ +"""Fixtures for the exec-sim harness. + +Registers all tools ONCE per session against: + +- a recording fake server (satisfies the MCPServer Protocol; stores + ``{name: (fn, annotations)}``), and +- a capture UE client that subclasses the REAL ``AsyncUnrealRemoteControl`` + but overrides only ``execute_python`` -- so client-side codegen delegation + (spawn_actor -> _CodeGen.spawn_actor_code, ...) is exercised verbatim while + every generated script is captured instead of sent over HTTP. + +Tests are plain sync functions; async tool coroutines are driven with +``asyncio.run`` (pytest-asyncio strict mode stays untouched). +""" + +from __future__ import annotations + +import asyncio +import sys +from pathlib import Path + +import pytest + +PROJECT_ROOT = Path(__file__).resolve().parents[2] +if str(PROJECT_ROOT) not in sys.path: + sys.path.insert(0, str(PROJECT_ROOT)) + +from remote_control.async_client import AsyncUnrealRemoteControl # noqa: E402 +from tests.exec_sim.registry import REGISTRY # noqa: E402 +from ue_mcp.tools import register_all_tools # noqa: E402 + + +class RecordingServer: + """Fake MCP server: records registrations instead of serving them.""" + + def __init__(self): + self.tools: dict[str, tuple] = {} + + def tool(self, *, name: str, description: str, annotations: dict | None = None): + def decorator(fn): + if name in self.tools: + raise AssertionError(f"duplicate tool registration: {name}") + self.tools[name] = (fn, annotations) + return fn + + return decorator + + +class CaptureUE(AsyncUnrealRemoteControl): + """Real client delegation, fake transport. + + Deliberately does NOT call super().__init__ -- no HTTP client, no circuit + breaker. The convenience methods (spawn_actor, delete_actor, list_actors, + set_actor_transform, find_assets, get_level_info, save_level) run their + real code paths and land in this overridden execute_python. + """ + + def __init__(self): # noqa: D107 -- see class docstring + self.captured: list[str] = [] + # Shape mirrors remote_control.execution._parse_result output. + self.result: dict = {"result": None, "output": "", "error": None} + + async def execute_python(self, code: str) -> dict: + self.captured.append(code) + return dict(self.result) + + +class Toolbox: + """Session-wide registration + code capture with per-tool caching.""" + + def __init__(self): + self.server = RecordingServer() + self.ue = CaptureUE() + register_all_tools(self.server, self.ue) + self._code_cache: dict[str, list[str]] = {} + + @property + def registered_names(self) -> set[str]: + return set(self.server.tools) + + def invoke(self, tool_name: str, **kwargs) -> list[str]: + """Invoke a registered tool coroutine; return the captured scripts.""" + fn, _annotations = self.server.tools[tool_name] + self.ue.captured = [] + asyncio.run(fn(**kwargs)) + return list(self.ue.captured) + + def codes_for(self, tool_name: str) -> list[str]: + """Captured scripts for the registry's canonical sentinel kwargs (cached).""" + if tool_name not in self._code_cache: + entry = REGISTRY[tool_name] + self._code_cache[tool_name] = self.invoke(tool_name, **entry.kwargs) + return self._code_cache[tool_name] + + +@pytest.fixture(scope="session") +def toolbox() -> Toolbox: + return Toolbox() diff --git a/tests/exec_sim/registry.py b/tests/exec_sim/registry.py new file mode 100644 index 0000000..dc081b1 --- /dev/null +++ b/tests/exec_sim/registry.py @@ -0,0 +1,381 @@ +"""Registry of every MCP tool registered by ``register_all_tools``. + +Completeness is gate #1: test_codegen_exec asserts that the keys of REGISTRY +equal the registered tool names exactly, so a future tool cannot dodge the +harness -- it must be classified here as CODEGEN or DIRECT. + +Modes: + +- CODEGEN: the tool (or the client method it delegates to) builds a UE editor + Python script and sends it through ``ue.execute_python``. Exec-simulated. +- DIRECT: no Python is generated (HTTP Remote Control object calls, HTTP to + the perception plugin, pure server-side data, or verbatim pass-through). + Skipped by the exec gates; the ``notes`` field carries the reason. + +Sentinel values are distinctive so the sentinel gate can assert each +``sentinel_checkable`` kwarg appears *literally* in the generated source +(raw or f-string-escaped) -- catching validated-then-discarded arguments. +Floats are dyadic (x.25 / x.4375 ...) so ``str()``/``json.dumps`` round-trip +exactly. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field + +CODEGEN = "CODEGEN" +DIRECT = "DIRECT" + +# Shared sentinels -- the stub's seeded world (unreal_stub._ACTOR_SPECS) matches +# these so tool success branches actually execute. +SENTINEL_LABEL = "SENTINEL_LBL_9Q" +SENTINEL_LABEL_B = "SENTINEL_LBL_B2" +SENTINEL_TAG = "SENTINEL_TAG_9Q" +SENTINEL_ACTOR_PATH = "/Game/Maps/TestMap.TestMap:PersistentLevel.SENTINEL_ACT_9Q" +SENTINEL_DIR = "/Game/SENTINEL_DIR_9Q" +SENTINEL_ASSET_PATH = SENTINEL_DIR + "/SENTINEL_ASSET_9Q" +SENTINEL_BP_PATH = SENTINEL_DIR + "/SENTINEL_BP_9Q" +SENTINEL_SEQ_PATH = SENTINEL_DIR + "/SENTINEL_SEQ_9Q" +SENTINEL_MAT_PATH = SENTINEL_DIR + "/SENTINEL_MAT_9Q" +SENTINEL_MAP_PATH = "/Game/Maps/SENTINEL_MAP_9Q" +SENTINEL_MESH_PATH = "/Game/Meshes/SENTINEL_MESH_9Q" +SENTINEL_FX_PATH = "/Game/FX/SENTINEL_FX_9Q" +SENTINEL_PROP = "SENTINEL_Prop_9Q" + + +@dataclass(frozen=True) +class ToolEntry: + tool_name: str + mode: str + kwargs: dict = field(default_factory=dict) + # kwarg names whose values must appear literally in the generated source + sentinel_checkable: tuple[str, ...] = () + # True when an honest error RESULT is the correct behavior under the + # default all-success stub (e.g. a not-implemented report). + expect_error: bool = False + notes: str = "" + + +_ENTRIES = [ + # ------------------------------------------------------------- actors.py + ToolEntry( + "ue_spawn_actor", CODEGEN, + kwargs=dict(class_name="StaticMeshActor", x=7317.25, y=811.25, z=97.25, + rx=14.25, ry=28.25, rz=42.25, label=SENTINEL_LABEL), + sentinel_checkable=("class_name", "x", "y", "z", "rx", "ry", "rz", "label"), + notes="codegen via client (_CodeGen.spawn_actor_code)", + ), + ToolEntry( + "ue_delete_actor", CODEGEN, + kwargs=dict(actor_path=SENTINEL_ACTOR_PATH), + sentinel_checkable=("actor_path",), + notes="codegen via client; actor-resolver contract gated in test_honesty", + ), + ToolEntry( + "ue_list_actors", CODEGEN, + kwargs=dict(class_filter="SENTINEL_Cls9Q"), + sentinel_checkable=("class_filter",), + notes="codegen via client (_CodeGen.list_actors_code)", + ), + ToolEntry( + "ue_set_transform", CODEGEN, + kwargs=dict(actor_path=SENTINEL_ACTOR_PATH, x=7317.25, y=811.25, z=97.25, + rx=14.25, ry=28.25, rz=42.25, sx=1.25, sy=2.25, sz=3.25), + sentinel_checkable=("actor_path", "x", "y", "z", "rx", "ry", "rz", "sx", "sy", "sz"), + notes="codegen via client; actor-resolver contract gated in test_honesty", + ), + ToolEntry( + "ue_duplicate_actor", CODEGEN, + kwargs=dict(actor_label=SENTINEL_LABEL, offset_x=433.25, offset_y=12.25, offset_z=99.25), + sentinel_checkable=("actor_label", "offset_x", "offset_y", "offset_z"), + ), + ToolEntry( + "ue_get_actor_bounds", CODEGEN, + kwargs=dict(actor_label=SENTINEL_LABEL), + sentinel_checkable=("actor_label",), + ), + # --------------------------------------------------------- properties.py + ToolEntry( + "ue_get_property", DIRECT, + notes="direct Remote Control HTTP property read (rc.get_property); no Python generated", + ), + ToolEntry( + "ue_set_property", DIRECT, + notes="direct Remote Control HTTP property write (rc.set_property); no Python generated", + ), + # -------------------------------------------------------- python_exec.py + ToolEntry( + "ue_execute_python", DIRECT, + notes="pass-through: executes caller-supplied code verbatim; nothing generated to verify", + ), + # ------------------------------------------------------------- assets.py + ToolEntry( + "ue_find_assets", CODEGEN, + kwargs=dict(search_pattern="SENTINELPAT9Q"), + sentinel_checkable=("search_pattern",), + notes="codegen via client; quote/backslash escaping gated in test_honesty. " + "class_filter is accepted but ignored by the current codegen (not gated here).", + ), + ToolEntry( + "ue_create_material", CODEGEN, + kwargs=dict(name="SENTINEL_MAT_9Q", base_color_r=0.11, base_color_g=0.23, + base_color_b=0.37, roughness=0.4375, metallic=0.8125), + sentinel_checkable=("name", "base_color_r", "base_color_g", "base_color_b", + "roughness", "metallic"), + ), + ToolEntry( + "ue_delete_asset", CODEGEN, + kwargs=dict(asset_path=SENTINEL_ASSET_PATH), + sentinel_checkable=("asset_path",), + ), + # -------------------------------------------------------------- level.py + ToolEntry("ue_save_level", CODEGEN, notes="codegen via client (_CodeGen.save_level_code)"), + ToolEntry("ue_get_level_info", CODEGEN, notes="codegen via client (_CodeGen.get_level_info_code)"), + ToolEntry( + "ue_load_level", CODEGEN, + kwargs=dict(level_path=SENTINEL_MAP_PATH), + sentinel_checkable=("level_path",), + notes="scripted-failure honesty contract in test_honesty (load_level -> False)", + ), + ToolEntry("ue_get_world_info", CODEGEN), + # ------------------------------------------------------------ mograph.py + ToolEntry( + "ue_create_cloner", CODEGEN, + kwargs=dict(layout="Circle", mesh_path=SENTINEL_MESH_PATH, + count_x=7317, count_y=6113, count_z=4231, spacing=433.25, + x=1.25, y=2.25, z=3.25, label=SENTINEL_LABEL), + sentinel_checkable=("layout", "mesh_path", "count_x", "count_y", "count_z", + "spacing", "x", "y", "z", "label"), + notes="sentinel gate is the arg-discard trap (layout/counts/spacing/mesh_path)", + ), + ToolEntry( + "ue_create_niagara_system", CODEGEN, + kwargs=dict(system_asset=SENTINEL_FX_PATH, x=4.25, y=5.25, z=6.25, label=SENTINEL_LABEL), + sentinel_checkable=("system_asset", "x", "y", "z", "label"), + ), + ToolEntry( + "ue_create_pcg_graph", CODEGEN, + kwargs=dict(x=1234.25, y=2345.25, z=3456.25, label=SENTINEL_LABEL), + sentinel_checkable=("x", "y", "z", "label"), + notes="extent_* excluded from sentinel gate: embedded transformed (extent/100), not literally", + ), + # --------------------------------------------------------- blueprints.py + ToolEntry( + "ue_create_blueprint", CODEGEN, + kwargs=dict(name="SENTINEL_BP_9Q", folder=SENTINEL_DIR, parent_class="Pawn"), + sentinel_checkable=("name", "folder", "parent_class"), + ), + ToolEntry( + "ue_add_component", CODEGEN, + kwargs=dict(actor_label=SENTINEL_LABEL, component_class="PointLightComponent", + component_name="SENTINEL_COMP_9Q"), + sentinel_checkable=("actor_label", "component_class", "component_name"), + ), + ToolEntry( + "ue_set_component_property", CODEGEN, + kwargs=dict(actor_label=SENTINEL_LABEL, component_class="StaticMeshComponent", + property_name=SENTINEL_PROP, value="7317.25"), + sentinel_checkable=("actor_label", "component_class", "property_name", "value"), + notes="success-branch exec is the bare-`true` trap", + ), + ToolEntry( + "ue_set_blueprint_defaults", CODEGEN, + kwargs=dict(blueprint_path=SENTINEL_BP_PATH, properties='{"SENTINEL_Prop_9Q": 7317}'), + sentinel_checkable=("blueprint_path", "properties"), + ), + ToolEntry( + "ue_compile_blueprint", CODEGEN, + kwargs=dict(blueprint_path=SENTINEL_BP_PATH), + sentinel_checkable=("blueprint_path",), + ), + ToolEntry( + "ue_get_actor_components", CODEGEN, + kwargs=dict(actor_label=SENTINEL_LABEL), + sentinel_checkable=("actor_label",), + ), + ToolEntry( + "ue_spawn_blueprint", CODEGEN, + kwargs=dict(blueprint_path=SENTINEL_BP_PATH, x=7.25, y=8.25, z=9.25, + rx=10.25, ry=11.25, rz=12.25, label=SENTINEL_LABEL), + sentinel_checkable=("blueprint_path", "x", "y", "z", "rx", "ry", "rz", "label"), + notes="label kwarg exercises the label_line indentation path", + ), + # --------------------------------------------------------- perception.py + ToolEntry( + "ue_viewport_percept", DIRECT, + notes="primary path is HTTP to the ViewportPerception plugin (:30011); the Python " + "fallback codegen (_fallback_capture) IS exec-simulated in " + "test_honesty::test_viewport_fallback_does_not_claim_success_with_empty_image", + ), + ToolEntry( + "ue_viewport_watch", DIRECT, + notes="HTTP-only control of the ViewportPerception plugin; no codegen", + ), + ToolEntry( + "ue_viewport_config", DIRECT, + notes="HTTP-only configuration of the ViewportPerception plugin; no codegen", + ), + ToolEntry( + "ue_viewport_diff", CODEGEN, + kwargs=dict(delay_ms=100), + notes="captures two identical snapshot scripts; delay_ms is a host-side sleep, " + "not embedded in the generated source", + ), + # -------------------------------------------------------------- scene.py + ToolEntry( + "ue_get_actor_details", CODEGEN, + kwargs=dict(actor_label=SENTINEL_LABEL), + sentinel_checkable=("actor_label",), + notes="success-branch exec is the phantom actor.is_hidden() trap", + ), + ToolEntry( + "ue_query_scene", CODEGEN, + kwargs=dict(tag_filter=SENTINEL_TAG, name_pattern=SENTINEL_LABEL, + near_x=101.25, near_y=202.25, near_z=303.25, + radius=9999.25, max_results=137), + sentinel_checkable=("tag_filter", "name_pattern", "near_x", "near_y", "near_z", + "radius", "max_results"), + notes="filters chosen to MATCH the stub world so the append branch executes", + ), + ToolEntry( + "ue_get_component_details", CODEGEN, + kwargs=dict(actor_label=SENTINEL_LABEL, component_name="StaticMeshComponent"), + sentinel_checkable=("actor_label", "component_name"), + ), + ToolEntry( + "ue_get_actor_hierarchy", CODEGEN, + kwargs=dict(actor_label=SENTINEL_LABEL), + sentinel_checkable=("actor_label",), + ), + # ------------------------------------------------------------ spatial.py + ToolEntry( + "ue_ground_trace", CODEGEN, + kwargs=dict(x=1234.25, y=5678.25, start_z=91011.25), + sentinel_checkable=("x", "y", "start_z"), + ), + ToolEntry( + "ue_snap_to_ground", CODEGEN, + kwargs=dict(actor_label=SENTINEL_LABEL, align_to_normal=True, z_offset=77.25), + sentinel_checkable=("actor_label", "z_offset"), + ), + ToolEntry( + "ue_spatial_query", CODEGEN, + kwargs=dict(mode="nearest", x=101.25, y=202.25, z=303.25, count=17), + sentinel_checkable=("mode", "x", "y", "z", "count"), + ), + ToolEntry( + "ue_measure", CODEGEN, + kwargs=dict(mode="distance", actor_a=SENTINEL_LABEL, actor_b=SENTINEL_LABEL_B), + sentinel_checkable=("mode", "actor_a", "actor_b"), + ), + # ----------------------------------------------------------- lighting.py + ToolEntry( + "ue_setup_sky_atmosphere", CODEGEN, + kwargs=dict(sun_elevation=47.25, sun_azimuth=133.25, sun_intensity=6.25, + fog=True, fog_density=0.0625, clouds=True), + sentinel_checkable=("sun_elevation", "sun_azimuth", "sun_intensity", "fog_density"), + notes="values embedded via json.dumps(settings) injected ahead of _RIG_CODE", + ), + ToolEntry( + "ue_set_time_of_day", CODEGEN, + kwargs=dict(hour=13.25), + sentinel_checkable=("hour",), + notes="hour surfaces literally as settings['_hour']", + ), + ToolEntry( + "ue_list_mood_presets", DIRECT, + notes="pure server-side preset catalog; never touches UE", + ), + ToolEntry( + "ue_apply_mood_preset", CODEGEN, + kwargs=dict(name="noir"), + notes="preset name resolved server-side; only derived preset values are embedded", + ), + ToolEntry( + "ue_blend_mood_presets", CODEGEN, + kwargs=dict(preset_a="golden_hour", preset_b="noir", t=0.25), + notes="blend inputs resolved server-side; only interpolated values are embedded", + ), + # ---------------------------------------------------------- materials.py + ToolEntry( + "ue_create_material_instance", CODEGEN, + kwargs=dict(name="SENTINEL_MI_9Q", parent_material=SENTINEL_MAT_PATH, folder=SENTINEL_DIR), + sentinel_checkable=("name", "parent_material", "folder"), + ), + ToolEntry( + "ue_set_material_parameter", CODEGEN, + kwargs=dict(material_path=SENTINEL_MAT_PATH, param_name="SENTINEL_Param_9Q", + value="0.4375", param_type="scalar"), + sentinel_checkable=("material_path", "param_name", "value"), + ), + ToolEntry( + "ue_get_material_parameters", CODEGEN, + kwargs=dict(material_path=SENTINEL_MAT_PATH), + sentinel_checkable=("material_path",), + notes="exec gate is the json.dumps-on-Name-keys trap", + ), + ToolEntry( + "ue_assign_material", CODEGEN, + kwargs=dict(actor_label=SENTINEL_LABEL, material_path=SENTINEL_MAT_PATH, slot_index=13), + sentinel_checkable=("actor_label", "material_path", "slot_index"), + ), + # ------------------------------------------------------------- editor.py + ToolEntry( + "ue_console_command", CODEGEN, + kwargs=dict(command="stat SENTINEL_9Q"), + sentinel_checkable=("command",), + ), + ToolEntry( + "ue_undo", DIRECT, + notes="honest not-implemented: returns an explanatory error without an editor " + "round-trip (no editor-transaction route in the UE Python API); " + "contract pinned in test_honesty", + ), + ToolEntry( + "ue_redo", DIRECT, + notes="honest not-implemented, same contract as ue_undo", + ), + ToolEntry( + "ue_focus_actor", CODEGEN, + kwargs=dict(actor_label=SENTINEL_LABEL), + sentinel_checkable=("actor_label",), + notes="scripted-failure honesty contract (LevelEditorSubsystem absent) in test_honesty", + ), + ToolEntry( + "ue_select_actors", CODEGEN, + kwargs=dict(actor_labels_json='["SENTINEL_LBL_9Q"]'), + sentinel_checkable=("actor_labels_json",), + notes="labels JSON embedded f-string-escaped; sentinel gate accepts the escaped form", + ), + # ---------------------------------------------------------- sequencer.py + ToolEntry( + "ue_create_level_sequence", CODEGEN, + kwargs=dict(name="SENTINEL_SEQ_9Q", folder=SENTINEL_DIR), + sentinel_checkable=("name", "folder"), + ), + ToolEntry( + "ue_play_sequence", CODEGEN, + kwargs=dict(sequence_path=SENTINEL_SEQ_PATH, start_time=3.25, playback_rate=1.25), + sentinel_checkable=("sequence_path", "start_time", "playback_rate"), + ), + ToolEntry( + "ue_add_actor_to_sequence", CODEGEN, + kwargs=dict(sequence_path=SENTINEL_SEQ_PATH, actor_label=SENTINEL_LABEL), + sentinel_checkable=("sequence_path", "actor_label"), + ), + ToolEntry( + "ue_add_keyframe", CODEGEN, + kwargs=dict(sequence_path=SENTINEL_SEQ_PATH, actor_label=SENTINEL_LABEL, + property_name=SENTINEL_PROP, time_seconds=4.25, value="7317.25"), + sentinel_checkable=("sequence_path", "actor_label", "property_name", + "time_seconds", "value"), + expect_error=True, + notes="honest not-implemented report: an error RESULT is the CORRECT behavior", + ), +] + +REGISTRY: dict[str, ToolEntry] = {e.tool_name: e for e in _ENTRIES} +assert len(REGISTRY) == len(_ENTRIES), "duplicate tool_name in registry" + +CODEGEN_TOOLS: list[str] = sorted(n for n, e in REGISTRY.items() if e.mode == CODEGEN) +DIRECT_TOOLS: list[str] = sorted(n for n, e in REGISTRY.items() if e.mode == DIRECT) diff --git a/tests/exec_sim/test_codegen_exec.py b/tests/exec_sim/test_codegen_exec.py new file mode 100644 index 0000000..5d92111 --- /dev/null +++ b/tests/exec_sim/test_codegen_exec.py @@ -0,0 +1,141 @@ +"""Exec-simulation gates for every CODEGEN tool. + +Gate 0 (registry): REGISTRY keys == registered tool names, exactly. +Gate 1 (compile): generated Python must compile. +Gate 2 (exec): generated Python must run to completion against the strict + ``unreal`` stub in its default all-success world, print a + RESULT: line the product parser understands, and that + RESULT must not report failure (unless the registry entry + declares an honest error is expected). +Gate 3 (sentinel): every ``sentinel_checkable`` kwarg value must appear + literally (raw or f-string-escaped) in the generated + source -- catching args that are validated then discarded. + +The 415-test mock suite asserts on code *strings*; these gates assert on code +*behavior*, which is what the historical liar/phantom/discard bugs slip past. +""" + +from __future__ import annotations + +import pytest + +from tests.exec_sim.registry import CODEGEN_TOOLS, DIRECT, REGISTRY +from tests.exec_sim.unreal_stub import ( + exec_generated, + has_result_line, + is_failure, + make_unreal_stub, + parse_result, +) +from ue_mcp.tools._validation import escape_for_fstring + +# -------------------------------------------------------------------------- +# Gate 0: completeness -- no tool can dodge the harness +# -------------------------------------------------------------------------- + + +def test_registry_covers_exactly_the_registered_tools(toolbox): + registered = toolbox.registered_names + in_registry = set(REGISTRY) + missing = sorted(registered - in_registry) + stale = sorted(in_registry - registered) + assert not missing and not stale, ( + f"registry drift -- unclassified registered tools: {missing}; " + f"registry entries with no registered tool: {stale}. " + "Every tool registered by register_all_tools needs a registry.py entry " + "classified CODEGEN or DIRECT." + ) + + +def test_direct_tools_carry_a_reason(): + unreasoned = [n for n, e in REGISTRY.items() if e.mode == DIRECT and not e.notes.strip()] + assert not unreasoned, f"DIRECT entries must explain why they are skipped: {unreasoned}" + + +def test_codegen_tools_actually_generate_code(toolbox): + silent = [name for name in CODEGEN_TOOLS if not toolbox.codes_for(name)] + assert not silent, ( + f"classified CODEGEN but captured no execute_python call: {silent} " + "(misclassified, or the tool errored before generating code)" + ) + + +# -------------------------------------------------------------------------- +# Gate 1: generated Python must compile +# -------------------------------------------------------------------------- + + +@pytest.mark.parametrize("tool_name", CODEGEN_TOOLS) +def test_generated_code_compiles(toolbox, tool_name): + codes = toolbox.codes_for(tool_name) + assert codes, f"{tool_name}: no generated code captured" + for i, code in enumerate(codes): + try: + compile(code, f"<{tool_name}#{i}>", "exec") + except SyntaxError as e: + pytest.fail( + f"{tool_name}: generated script #{i} does not compile: {e}\n" + f"--- generated source ---\n{code}" + ) + + +# -------------------------------------------------------------------------- +# Gate 2: success branch must execute and report honestly +# -------------------------------------------------------------------------- + + +@pytest.mark.parametrize("tool_name", CODEGEN_TOOLS) +def test_generated_code_executes_and_reports(toolbox, tool_name): + entry = REGISTRY[tool_name] + codes = toolbox.codes_for(tool_name) + stub = make_unreal_stub() # default: everything the real API offers succeeds + + for i, code in enumerate(codes): + try: + stdout = exec_generated(code, stub, name=f"<{tool_name}#{i}>") + except SyntaxError as e: + pytest.fail(f"{tool_name}: script #{i} does not compile (see compile gate): {e}") + except Exception as e: # noqa: BLE001 -- any runtime crash is the finding + pytest.fail( + f"{tool_name}: script #{i} crashed during exec against the strict " + f"unreal stub: {type(e).__name__}: {e}\n" + f"--- generated source ---\n{code}" + ) + + assert has_result_line(stdout), ( + f"{tool_name}: script #{i} printed no RESULT: line. stdout was:\n{stdout!r}" + ) + parsed = parse_result(stdout) + + if entry.expect_error: + continue # an honest error is this tool's documented correct behavior + + assert not is_failure(parsed["result"]), ( + f"{tool_name}: script #{i} reported failure under the all-success stub " + f"(a tool that can never succeed is dead, or it hit a bug in its own " + f"success branch): RESULT={parsed['result']!r}" + ) + + +# -------------------------------------------------------------------------- +# Gate 3: sentinel kwargs must survive into the generated source +# -------------------------------------------------------------------------- + + +@pytest.mark.parametrize("tool_name", CODEGEN_TOOLS) +def test_sentinel_kwargs_reach_the_generated_source(toolbox, tool_name): + entry = REGISTRY[tool_name] + if not entry.sentinel_checkable: + pytest.skip(f"{tool_name}: no literally-embedded kwargs to check") + + blob = "\n".join(toolbox.codes_for(tool_name)) + missing = [] + for kwarg in entry.sentinel_checkable: + value = str(entry.kwargs[kwarg]) + if value not in blob and escape_for_fstring(value) not in blob: + missing.append(f"{kwarg}={value!r}") + + assert not missing, ( + f"{tool_name}: kwargs validated but absent from the generated source " + f"(validated-then-discarded): {missing}" + ) diff --git a/tests/exec_sim/test_honesty.py b/tests/exec_sim/test_honesty.py new file mode 100644 index 0000000..09f26ba --- /dev/null +++ b/tests/exec_sim/test_honesty.py @@ -0,0 +1,266 @@ +"""Scripted-failure honesty contracts for the historical liars. + +Each test forces a failure the real editor can produce (level fails to load, +focus API absent, no image captured, ...) and asserts the generated code does +not claim success anyway. Complements test_codegen_exec, which only checks the +all-success world. +""" + +from __future__ import annotations + +import asyncio +import json +import os +import tempfile + +import pytest + +from tests.exec_sim.registry import ( + REGISTRY, + SENTINEL_ACTOR_PATH, + SENTINEL_LABEL, + SENTINEL_MAP_PATH, + SENTINEL_MAT_PATH, + SENTINEL_PROP, +) +from tests.exec_sim.unreal_stub import exec_generated, make_unreal_stub, parse_result + + +def _single_code(toolbox, tool_name, **kwargs) -> str: + codes = toolbox.invoke(tool_name, **kwargs) if kwargs else toolbox.codes_for(tool_name) + assert codes, f"{tool_name}: no generated code captured" + return codes[-1] + + +# -------------------------------------------------------------------------- +# level.py -- ue_load_level must not report loaded:true when the load failed +# -------------------------------------------------------------------------- + + +def test_load_level_failure_is_not_reported_as_loaded(toolbox): + code = _single_code(toolbox, "ue_load_level", level_path=SENTINEL_MAP_PATH) + stub = make_unreal_stub(load_level=False) # EditorLevelLibrary.load_level -> False + result = parse_result(exec_generated(code, stub, name=""))["result"] + assert isinstance(result, dict), f"expected dict RESULT, got {result!r}" + assert not result.get("loaded"), ( + f"load_level() returned False but the generated code claimed success: {result!r}" + ) + + +# -------------------------------------------------------------------------- +# editor.py -- ue_focus_actor must not claim focused when the focus path is absent +# -------------------------------------------------------------------------- + + +def test_focus_actor_does_not_claim_focus_without_a_focus_api(toolbox): + code = _single_code(toolbox, "ue_focus_actor", actor_label=SENTINEL_LABEL) + stub = make_unreal_stub(level_editor_subsystem=False) # no LevelEditorSubsystem at all + result = parse_result(exec_generated(code, stub, name=""))["result"] + assert not (isinstance(result, dict) and result.get("focused")), ( + f"no viewport-focus API exists in this world, yet the code claimed focus: {result!r}" + ) + + +# -------------------------------------------------------------------------- +# editor.py -- undo/redo are honest not-implemented stubs +# (no editor-transaction route exists in the UE Python API — verified 5.7, and +# Epic's 5.8 MCP surface ships none either; the tools must say so up front +# instead of probing phantom APIs or claiming success) +# -------------------------------------------------------------------------- + + +@pytest.mark.parametrize("tool_name", ["ue_undo", "ue_redo"]) +def test_undo_redo_report_not_implemented(toolbox, tool_name): + fn, _annotations = toolbox.server.tools[tool_name] + toolbox.ue.captured = [] + result = json.loads(asyncio.run(fn())) + assert not toolbox.ue.captured, ( + f"{tool_name}: sent code to the editor despite having no real API route" + ) + assert isinstance(result, dict) and "not implemented" in str(result.get("error", "")).lower(), ( + f"{tool_name}: expected an explicit not-implemented error, got {result!r}" + ) + + +# -------------------------------------------------------------------------- +# blueprints.py -- set_component_property's success dict must be valid Python +# -------------------------------------------------------------------------- + + +def test_set_component_property_success_branch_is_valid_python(toolbox): + code = _single_code( + toolbox, "ue_set_component_property", + actor_label=SENTINEL_LABEL, component_class="StaticMeshComponent", + property_name=SENTINEL_PROP, value="7317.25", + ) + stub = make_unreal_stub() + result = parse_result(exec_generated(code, stub, name=""))["result"] + assert isinstance(result, dict) and result.get("set") is True, ( + f"the set succeeded but the success dict did not evaluate cleanly " + f"(bare JSON `true` instead of Python True?): RESULT={result!r}" + ) + + +# -------------------------------------------------------------------------- +# blueprints.py -- spawn_blueprint with a label must still compile +# -------------------------------------------------------------------------- + + +def test_spawn_blueprint_with_label_compiles(toolbox): + code = _single_code(toolbox, "ue_spawn_blueprint", **REGISTRY["ue_spawn_blueprint"].kwargs) + try: + compile(code, "", "exec") + except SyntaxError as e: + pytest.fail( + f"spawn_blueprint with label generates non-compiling Python " + f"(label_line indentation): {e}\n--- generated source ---\n{code}" + ) + + +# -------------------------------------------------------------------------- +# mograph.py -- cloner arguments must reach the generated code +# -------------------------------------------------------------------------- + + +def test_cloner_arguments_are_not_discarded(toolbox): + entry = REGISTRY["ue_create_cloner"] + code = "\n".join(toolbox.codes_for("ue_create_cloner")) + missing = [ + k for k in ("layout", "mesh_path", "count_x", "count_y", "count_z", "spacing") + if str(entry.kwargs[k]) not in code + ] + assert not missing, ( + f"ue_create_cloner validates these args, then generates code that ignores them: " + f"{missing} (a cloner with no layout/counts/spacing/mesh is set dressing theater)" + ) + + +# -------------------------------------------------------------------------- +# assets.py -- find_assets must survive quotes and backslashes in the pattern +# -------------------------------------------------------------------------- + + +def test_find_assets_pattern_with_quote_and_backslash_still_compiles(toolbox): + hostile = 'SENT"INEL\\9Q' + code = _single_code(toolbox, "ue_find_assets", search_pattern=hostile) + try: + compile(code, "", "exec") + except SyntaxError as e: + pytest.fail( + f"a search pattern containing a quote/backslash breaks the generated " + f"script (unescaped f-string interpolation): {e}\n" + f"--- generated source ---\n{code}" + ) + + +# -------------------------------------------------------------------------- +# remote_control/codegen.py -- level actors are not assets +# -------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + ("tool_name", "kwargs"), + [ + ("ue_delete_actor", dict(actor_path=SENTINEL_ACTOR_PATH)), + ("ue_set_transform", dict(actor_path=SENTINEL_ACTOR_PATH, x=1.25, y=2.25, z=3.25)), + ], +) +def test_level_actor_resolution_does_not_use_the_asset_api(toolbox, tool_name, kwargs): + code = _single_code(toolbox, tool_name, **kwargs) + assert "EditorAssetLibrary.load_asset" not in code, ( + f"{tool_name}: resolves a LEVEL actor via unreal.EditorAssetLibrary.load_asset -- " + "that loads content-browser assets and returns garbage-or-None for actor object " + "paths, so the tool silently no-ops in the real editor. Resolve through the level " + "(e.g. EditorActorSubsystem.get_all_level_actors + path match)." + ) + assert SENTINEL_ACTOR_PATH in code, f"{tool_name}: actor_path vanished from the generated code" + + +# -------------------------------------------------------------------------- +# materials.py -- get_material_parameters must not blow up on json.dumps +# -------------------------------------------------------------------------- + + +def test_get_material_parameters_output_is_json_serializable(toolbox): + code = _single_code(toolbox, "ue_get_material_parameters", material_path=SENTINEL_MAT_PATH) + stub = make_unreal_stub() # parameter names are unreal.Name objects, as in the editor + try: + stdout = exec_generated(code, stub, name="") + except TypeError as e: + pytest.fail( + f"generated code crashed serializing its own result " + f"(unreal.Name dict keys passed straight to json.dumps?): {e}" + ) + result = parse_result(stdout)["result"] + assert isinstance(result, dict) and "parameters" in result, f"unexpected RESULT: {result!r}" + + +# -------------------------------------------------------------------------- +# scene.py -- get_actor_details is covered by the generic exec gate +# (phantom actor.is_hidden(); the strict actor stub exposes `hidden` instead), +# but pin the contract here so the fix is visible in the honesty suite too. +# -------------------------------------------------------------------------- + + +def test_get_actor_details_does_not_call_phantom_is_hidden(toolbox): + code = _single_code(toolbox, "ue_get_actor_details", actor_label=SENTINEL_LABEL) + stub = make_unreal_stub() + try: + stdout = exec_generated(code, stub, name="") + except AttributeError as e: + pytest.fail( + f"generated code calls an API that does not exist on actors: {e} " + "(UE actors expose the `hidden` attribute, not an is_hidden() method)" + ) + result = parse_result(stdout)["result"] + assert isinstance(result, dict) and not result.get("error"), f"unexpected RESULT: {result!r}" + + +# -------------------------------------------------------------------------- +# perception.py -- the Python fallback must not report success with no image +# -------------------------------------------------------------------------- + + +def test_viewport_fallback_does_not_claim_success_with_empty_image(monkeypatch): + """The primary ue_viewport_percept path is HTTP (DIRECT in the registry). + The Python fallback is trigger -> poll -> read across SEPARATE editor + executions (take_high_res_screenshot completes on a later frame). Simulate + an editor where the screenshot never lands by exec-ing every script the + tool sends, and assert the final payload flags the miss instead of + claiming a capture.""" + from ue_mcp.tools import perception + + monkeypatch.setattr(perception, "FALLBACK_POLL_ATTEMPTS", 3) + monkeypatch.setattr(perception, "FALLBACK_POLL_INTERVAL_S", 0.0) + + # Nothing writes a screenshot in this world; clear any stale capture file. + out_path = os.path.join(tempfile.gettempdir(), "ue_perception_capture.jpeg") + if os.path.exists(out_path): + os.remove(out_path) + + stub = make_unreal_stub(screenshot_writes_file=False) + + class EditorSim: + """Executes every script the tool sends, like the real editor would.""" + + def __init__(self): + self.scripts: list[str] = [] + + async def execute_python(self, code: str) -> dict: + self.scripts.append(code) + return parse_result(exec_generated(code, stub, name=f"")) + + ue = EditorSim() + final = asyncio.run(perception._fallback_capture(ue, 320, 200, "jpeg")) + + assert len(ue.scripts) >= 2, ( + "fallback regressed to a single editor execution — the screenshot file " + "can never exist in the same exec that triggered it" + ) + result = final.get("result") + assert isinstance(result, dict), f"expected dict result, got {final!r}" + assert result.get("image") == "", "no screenshot existed; image must be empty" + assert result.get("capture_status") in {"timeout", "trigger_failed"}, ( + "empty image must be flagged via capture_status, got " + f"{result.get('capture_status')!r} in {({k: v for k, v in result.items() if k != 'image'})!r}" + ) diff --git a/tests/exec_sim/unreal_stub.py b/tests/exec_sim/unreal_stub.py new file mode 100644 index 0000000..429e3a7 --- /dev/null +++ b/tests/exec_sim/unreal_stub.py @@ -0,0 +1,809 @@ +r"""Fake ``unreal`` module for exec-simulating generated UE5 editor Python. + +``make_unreal_stub()`` builds a fresh module object that is installed as +``sys.modules["unreal"]`` while a generated script executes (``exec_generated``). + +Strict by default: any top-level symbol not in the curated table raises +AttributeError -- that is how phantom-API bugs surface. The table was seeded by +grepping ``unreal\.`` across ue_mcp/tools/ and remote_control/codegen.py. + +Deliberately-phantom symbols (absent because the real UE 5.7 Python API does +not expose them -- do NOT add these): + +- ``EditorLevelLibrary.editor_undo`` / ``editor_redo`` +- ``SystemLibrary.transaction_undo`` / ``transaction_redo`` +- ``.is_hidden()`` (actors expose the ``hidden`` *attribute* instead) + +Scriptable outcomes (honesty tests force failure paths):: + + stub = make_unreal_stub(load_level=False) + stub.configure(ground_hit=False) + +Default world: three level actors are seeded, labelled ``SENTINEL_LBL_9Q`` +(tagged ``SENTINEL_TAG_9Q``), ``SENTINEL_LBL_B2`` and ``Cube_1`` -- matching +the sentinel kwargs in registry.py so tool success branches actually execute. +""" + +from __future__ import annotations + +import io +import sys +import types +from contextlib import contextmanager, redirect_stdout + +# Product parser: keeps this harness aligned with the real RESULT-line grammar. +from remote_control.execution import _parse_result + +# --------------------------------------------------------------------------- +# Value types (shared across stub instances -- they hold no config) +# --------------------------------------------------------------------------- + + +class Vector: + def __init__(self, x=0.0, y=0.0, z=0.0): + self.x, self.y, self.z = float(x), float(y), float(z) + + def __sub__(self, o): + return Vector(self.x - o.x, self.y - o.y, self.z - o.z) + + def __add__(self, o): + return Vector(self.x + o.x, self.y + o.y, self.z + o.z) + + def length(self): + return (self.x**2 + self.y**2 + self.z**2) ** 0.5 + + def __repr__(self): + return f"Vector({self.x}, {self.y}, {self.z})" + + +class Vector4: + def __init__(self, x=0.0, y=0.0, z=0.0, w=0.0): + self.x, self.y, self.z, self.w = float(x), float(y), float(z), float(w) + + +class Rotator: + def __init__(self, pitch=0.0, yaw=0.0, roll=0.0): + self.pitch, self.yaw, self.roll = float(pitch), float(yaw), float(roll) + + +class LinearColor: + def __init__(self, r=0.0, g=0.0, b=0.0, a=1.0): + self.r, self.g, self.b, self.a = float(r), float(g), float(b), float(a) + + +class Color: + def __init__(self, r=0, g=0, b=0, a=255): + self.r, self.g, self.b, self.a = int(r), int(g), int(b), int(a) + + +class Name: + """Non-str name type, like unreal.Name. json.dumps must NOT accept it as a + dict key -- exactly the trap that real parameter-name lists set.""" + + def __init__(self, s: str): + self._s = s + + def __str__(self): + return self._s + + def __repr__(self): + return f"Name({self._s!r})" + + def __eq__(self, other): + return str(self) == str(other) + + def __hash__(self): + return hash(self._s) + + +class _ClassInfo: + """What actor.get_class() / comp.get_class() returns.""" + + def __init__(self, name: str): + self._name = name + + def get_name(self): + return self._name + + +class _UEObject: + """Base for seeded UE classes (factories, filters, proxies, CDOs...). + + Instances accept any ctor args and support the generic property protocol. + Attribute access beyond this whitelist raises AttributeError (strict). + """ + + def __init__(self, *args, **kwargs): + self._props: dict = {} + + def set_editor_property(self, name, value): + self._props[str(name)] = value + return True + + def get_editor_property(self, name): + return self._props.get(str(name), 1.0) + + def get_name(self): + return type(self).__name__ + + def get_path_name(self): + return f"/Game/Stub/{type(self).__name__}" + + def get_class(self): + return _ClassInfo(type(self).__name__) + + +# --------------------------------------------------------------------------- +# Config +# --------------------------------------------------------------------------- + +_CONFIG_DEFAULTS = { + # EditorLevelLibrary.load_level return value. + "load_level": True, + # Whether load_asset / load_blueprint_class / does_asset_exist find things. + "load_asset": True, + # Whether SystemLibrary.line_trace_single hits anything. + "ground_hit": True, + # Whether unreal.LevelEditorSubsystem exists at all (viewport-focus path). + "level_editor_subsystem": True, + # Whether spawn_actor_from_class succeeds. + "spawn_actor": True, + # Whether AutomationLibrary.take_high_res_screenshot writes the file. + "screenshot_writes_file": False, +} + + +class _Config: + def __init__(self, **overrides): + self.__dict__.update(_CONFIG_DEFAULTS) + self.update(**overrides) + + def update(self, **kw): + for k, v in kw.items(): + if k not in _CONFIG_DEFAULTS: + raise TypeError(f"unknown stub config key: {k!r}") + setattr(self, k, v) + + +# Default seeded level actors: (label, class_name, path, tags, location) +_ACTOR_SPECS = ( + ( + "SENTINEL_LBL_9Q", + "StaticMeshActor", + "/Game/Maps/TestMap.TestMap:PersistentLevel.SENTINEL_ACT_9Q", + ("SENTINEL_TAG_9Q",), + (0.0, 0.0, 0.0), + ), + ( + "SENTINEL_LBL_B2", + "PointLight", + "/Game/Maps/TestMap.TestMap:PersistentLevel.SENTINEL_ACT_B2", + (), + (100.0, 200.0, 300.0), + ), + ( + "Cube_1", + "StaticMeshActor", + "/Game/Maps/TestMap.TestMap:PersistentLevel.Cube_1", + (), + (50.0, 0.0, 0.0), + ), +) + +# UE class names the generated code references by attribute (spawnable classes, +# component classes, asset classes, factories, expression nodes, proxies). +_SEEDED_CLASS_NAMES = ( + # actors / spawnables + "Actor", "Pawn", "Character", "StaticMeshActor", "PointLight", "SpotLight", + "CameraActor", "NiagaraActor", + "DirectionalLight", "SkyAtmosphere", "SkyLight", "ExponentialHeightFog", + "VolumetricCloud", "PostProcessVolume", + # components + "ActorComponent", "SceneComponent", "StaticMeshComponent", + "SkeletalMeshComponent", "PointLightComponent", "SpotLightComponent", + "AudioComponent", "BoxComponent", "SphereComponent", "NiagaraComponent", + "DecalComponent", "DirectionalLightComponent", "SkyLightComponent", + "ExponentialHeightFogComponent", + # asset classes + "Material", "MaterialInstanceConstant", "LevelSequence", + "MaterialExpressionConstant3Vector", "MaterialExpressionConstant", + # factories / misc instantiables + "MaterialFactoryNew", "MaterialInstanceConstantFactoryNew", + "BlueprintFactory", "LevelSequenceFactoryNew", "ARFilter", + "SequencerBindingProxy", +) + + +# --------------------------------------------------------------------------- +# Factory +# --------------------------------------------------------------------------- + + +def make_unreal_stub(**overrides) -> types.ModuleType: + """Build a fresh, isolated fake ``unreal`` module. See module docstring.""" + config = _Config(**overrides) + + # -- objects with per-stub state -------------------------------------- + + class _Component(_UEObject): + _PROPERTY_VALUES = {"intensity": 5000.0} + + def __init__(self, name="StaticMeshComponent", class_name=None): + super().__init__() + self._name = name + self._class_name = class_name or name + self.static_mesh = _Asset("/Game/Meshes/SM_StubMesh") + self.intensity = 5000.0 + self.light_color = LinearColor(1.0, 1.0, 1.0, 1.0) + + def get_name(self): + return self._name + + def get_class(self): + return _ClassInfo(self._class_name) + + def get_editor_property(self, name): + name = str(name) + if name == "light_color": + return LinearColor(1.0, 1.0, 1.0, 1.0) + if name == "settings": + return self._props.setdefault("settings", _UEObject()) + if name in self._PROPERTY_VALUES: + return self._PROPERTY_VALUES[name] + return super().get_editor_property(name) + + def get_num_materials(self): + return 1 + + def get_material(self, index): + return _Asset("/Game/Materials/M_Stub") + + def set_material(self, index, material): + return True + + def set_asset(self, asset): + return True + + def k2_attach_to(self, parent, *args, **kwargs): + return True + + def set_light_color(self, color): + return True + + def recapture_sky(self): + return True + + def recapture(self): + return True + + class _Asset(_UEObject): + """A loadable / creatable asset (material, blueprint, sequence...).""" + + def __init__(self, path="/Game/Stub/Asset"): + super().__init__() + self._path = path + + def get_path_name(self): + return self._path + + def get_name(self): + return self._path.rsplit("/", 1)[-1] + + def generated_class(self): + return type(self.get_name() + "_C", (_UEObject,), {}) + + def add_possessable(self, actor): + return _Binding() + + class _Binding: + def get_id(self): + return "BIND-0000-STUB" + + class _Actor: + """Level actor. STRICT: no ``is_hidden()`` -- ``hidden`` attr instead.""" + + def __init__(self, label, class_name="StaticMeshActor", path=None, + tags=(), location=(0.0, 0.0, 0.0)): + self._label = label + self._class_name = class_name + self._path = path or f"/Game/Maps/TestMap.TestMap:PersistentLevel.{label}" + self.tags = [Name(t) for t in tags] + self._location = Vector(*location) + self._rotation = Rotator() + self._scale = Vector(1.0, 1.0, 1.0) + self._components = [_Component("StaticMeshComponent")] + self.hidden = False + self.root_component = self._components[0] + self._props: dict = {} + + # identity + def get_actor_label(self): + return self._label + + def set_actor_label(self, label): + self._label = str(label) + + def get_name(self): + return self._label + + def get_path_name(self): + return self._path + + def get_class(self): + return _ClassInfo(self._class_name) + + # transform + def get_actor_location(self): + return self._location + + def set_actor_location(self, v, *args): + self._location = v + return True + + def get_actor_rotation(self): + return self._rotation + + def set_actor_rotation(self, r, *args): + self._rotation = r + return True + + def get_actor_scale3d(self): + return self._scale + + def set_actor_scale3d(self, v): + self._scale = v + return True + + def get_actor_bounds(self, only_colliding, *args): + return (Vector(0.0, 0.0, 100.0), Vector(50.0, 50.0, 100.0)) + + # components / hierarchy + def get_components_by_class(self, cls): + return list(self._components) + + def get_component_by_class(self, cls): + return self._components[0] + + def get_attach_parent_actor(self): + return None + + def get_attached_actors(self): + return [] + + # generic property protocol + def set_editor_property(self, name, value): + self._props[str(name)] = value + return True + + def get_editor_property(self, name): + name = str(name) + if name == "settings": + return self._props.setdefault("settings", _UEObject()) + return self._props.get(name, 1.0) + + world_actors = [_Actor(*spec) for spec in _ACTOR_SPECS] + _selection: list = [] + + def _find_actor_by_path(path): + for a in world_actors: + if a.get_path_name() == path: + return a + return None + + class _WorldSettings(_UEObject): + def get_editor_property(self, name): + if str(name) == "DefaultGameMode": + return _ClassInfo("GameModeBase") + return super().get_editor_property(name) + + class _World: + def get_name(self): + return "TestLevel" + + def get_path_name(self): + return "/Game/Maps/TestMap.TestMap" + + def get_streaming_levels(self): + return [] + + def get_world_settings(self): + return _WorldSettings() + + world = _World() + + class _HitResult: + def __init__(self): + self.impact_point = Vector(0.0, 0.0, 12.5) + self.impact_normal = Vector(0.0, 0.0, 1.0) + self.distance = 123.5 + self.hit_actor = world_actors[2] + self.blocking_hit = True + + # -- subsystems (all-static so both class-level and instance-level calls + # work; get_editor_subsystem simply returns the class) ---------------- + + class EditorActorSubsystem: + @staticmethod + def get_all_level_actors(): + return list(world_actors) + + @staticmethod + def spawn_actor_from_class(cls, location, rotation, *args, **kwargs): + if not config.spawn_actor: + return None + name = getattr(cls, "__name__", "Spawned") + actor = _Actor( + f"{name}_1", class_name=name, + path=f"/Game/Maps/TestMap.TestMap:PersistentLevel.{name}_1", + ) + world_actors.append(actor) + return actor + + @staticmethod + def destroy_actor(actor): + if actor in world_actors: + world_actors.remove(actor) + return True + + @staticmethod + def set_selected_level_actors(actors): + _selection[:] = list(actors) + + @staticmethod + def get_selected_level_actors(): + return list(_selection) + + @staticmethod + def duplicate_selected_actors(): + return [_Actor(a.get_actor_label() + "2") for a in _selection] or [_Actor("Dup_1")] + + class UnrealEditorSubsystem: + @staticmethod + def get_editor_world(): + return world + + @staticmethod + def get_level_viewport_camera_info(): + return (Vector(0.0, -500.0, 250.0), Rotator(-15.0, 90.0, 0.0)) + + class LevelEditorSubsystem: + @staticmethod + def focus_on_selected_actors(): + return True + + class LevelSequenceEditorSubsystem: + pass + + # -- static libraries (STRICT whitelists) ------------------------------ + + class EditorLevelLibrary: + # Deliberately ABSENT: editor_undo, editor_redo (phantom APIs). + @staticmethod + def get_editor_world(): + return world + + @staticmethod + def save_current_level(): + return True + + @staticmethod + def load_level(path): + return config.load_level + + @staticmethod + def set_selected_level_actors(actors): + _selection[:] = list(actors) + + @staticmethod + def get_actor_reference(path): + return _find_actor_by_path(path) or world_actors[0] + + class SystemLibrary: + # Deliberately ABSENT: transaction_undo, transaction_redo (phantom APIs). + @staticmethod + def execute_console_command(world_ctx, command): + return None + + @staticmethod + def line_trace_single(world_ctx, start, end, query, complex_trace, + ignore, draw, ret, *args, **kwargs): + return _HitResult() if config.ground_hit else None + + class EditorAssetLibrary: + @staticmethod + def load_asset(path): + return _Asset(str(path)) if config.load_asset else None + + @staticmethod + def load_blueprint_class(path): + if not config.load_asset: + return None + return type("StubBPClass", (_UEObject,), {}) + + @staticmethod + def does_asset_exist(path): + return bool(config.load_asset) + + @staticmethod + def save_asset(path, *args, **kwargs): + return True + + @staticmethod + def delete_asset(path): + return True + + class _AssetTools: + @staticmethod + def create_asset(asset_name, package_path, asset_class, factory): + return _Asset(f"{package_path}/{asset_name}") + + class AssetToolsHelpers: + @staticmethod + def get_asset_tools(): + return _AssetTools() + + class _AssetData: + def __init__(self): + self.asset_name = Name("SM_StubCube") + self.package_name = Name("/Game/Meshes/SM_StubCube") + self.asset_class_path = types.SimpleNamespace(asset_name=Name("StaticMesh")) + + class _AssetRegistry: + @staticmethod + def get_assets_by_package_name(name, *args, **kwargs): + return [] + + @staticmethod + def get_all_assets(ar_filter, *args, **kwargs): + return [_AssetData()] + + class AssetRegistryHelpers: + @staticmethod + def get_asset_registry(): + return _AssetRegistry() + + class BlueprintEditorLibrary: + @staticmethod + def compile_blueprint(bp): + return True + + class MaterialEditingLibrary: + @staticmethod + def create_material_expression(material, expression_class, x=0, y=0): + return _UEObject() + + @staticmethod + def connect_material_property(node, output_name, material_property): + return True + + @staticmethod + def recompile_material(material): + return True + + @staticmethod + def set_material_instance_scalar_parameter_value(mi, name, value): + return True + + @staticmethod + def set_material_instance_vector_parameter_value(mi, name, value): + return True + + @staticmethod + def set_material_instance_texture_parameter_value(mi, name, value): + return True + + # Parameter-name lists are unreal.Name objects in the real API (NOT str) + # -- json.dumps on a dict keyed by them must raise, as in the editor. + @staticmethod + def get_scalar_parameter_names(asset): + return [Name("StubScalarParam")] + + @staticmethod + def get_vector_parameter_names(asset): + return [Name("StubVectorParam")] + + @staticmethod + def get_texture_parameter_names(asset): + return [Name("StubTextureParam")] + + @staticmethod + def get_material_instance_scalar_parameter_value(asset, name): + return 0.5 + + @staticmethod + def get_material_instance_vector_parameter_value(asset, name): + return LinearColor(0.1, 0.2, 0.3, 1.0) + + @staticmethod + def get_material_instance_texture_parameter_value(asset, name): + return _Asset("/Game/Textures/T_Stub") + + # Default-value getters (base Material family — real 5.x API names). + @staticmethod + def get_material_default_scalar_parameter_value(material, name): + return 0.25 + + @staticmethod + def get_material_default_vector_parameter_value(material, name): + return LinearColor(0.4, 0.5, 0.6, 1.0) + + @staticmethod + def get_material_default_texture_parameter_value(material, name): + return _Asset("/Game/Textures/T_StubDefault") + + class AutomationLibrary: + @staticmethod + def take_high_res_screenshot(width, height, path, *args, **kwargs): + if config.screenshot_writes_file: + with open(path, "wb") as f: + f.write(b"\xff\xd8stub-image-bytes\xff\xd9") + return True + + class EditorUtilityLibrary: + @staticmethod + def get_selected_assets(): + return [] + + class MathLibrary: + @staticmethod + def make_rot_from_z(v): + return Rotator() + + class LevelSequenceEditorBlueprintLibrary: + @staticmethod + def open_level_sequence(seq): + return True + + @staticmethod + def set_current_time(t): + return True + + @staticmethod + def play(): + return True + + @staticmethod + def get_bound_objects(binding_proxy): + return [] + + # -- module-level functions --------------------------------------------- + + def get_editor_subsystem(cls): + return cls + + def find_class(name): + return type(str(name), (_UEObject,), {}) + + def load_class(outer, path): + return type(str(path).rsplit(".", 1)[-1], (_UEObject,), {}) + + def new_object(cls, outer=None, name=None, *args, **kwargs): + return _Component(str(name) if name else getattr(cls, "__name__", "Comp")) + + def get_default_object(cls): + return _UEObject() + + def load_asset(path): + return EditorAssetLibrary.load_asset(path) + + def load_object(outer, path): + return _find_actor_by_path(str(path)) or (_Asset(str(path)) if config.load_asset else None) + + def find_object(outer, path): + return _find_actor_by_path(str(path)) + + # -- assemble the module ------------------------------------------------ + + mod = types.ModuleType("unreal") + mod.__doc__ = "exec-sim stub for the UE5 editor Python API (tests/exec_sim)" + + for cls_name in _SEEDED_CLASS_NAMES: + setattr(mod, cls_name, type(cls_name, (_UEObject,), {})) + + exported = { + "Vector": Vector, "Vector4": Vector4, "Rotator": Rotator, + "LinearColor": LinearColor, "Color": Color, "Name": Name, + "EditorActorSubsystem": EditorActorSubsystem, + "UnrealEditorSubsystem": UnrealEditorSubsystem, + "LevelEditorSubsystem": LevelEditorSubsystem, + "LevelSequenceEditorSubsystem": LevelSequenceEditorSubsystem, + "EditorLevelLibrary": EditorLevelLibrary, + "SystemLibrary": SystemLibrary, + "EditorAssetLibrary": EditorAssetLibrary, + "AssetToolsHelpers": AssetToolsHelpers, + "AssetRegistryHelpers": AssetRegistryHelpers, + "BlueprintEditorLibrary": BlueprintEditorLibrary, + "MaterialEditingLibrary": MaterialEditingLibrary, + "AutomationLibrary": AutomationLibrary, + "EditorUtilityLibrary": EditorUtilityLibrary, + "MathLibrary": MathLibrary, + "LevelSequenceEditorBlueprintLibrary": LevelSequenceEditorBlueprintLibrary, + "MaterialProperty": types.SimpleNamespace( + MP_BASE_COLOR=0, MP_ROUGHNESS=1, MP_METALLIC=2, MP_EMISSIVE_COLOR=3, + ), + "TraceTypeQuery": types.SimpleNamespace(TRACE_TYPE_QUERY1=1, TRACE_TYPE_QUERY2=2), + "DrawDebugTrace": types.SimpleNamespace(NONE=0), + "get_editor_subsystem": get_editor_subsystem, + "find_class": find_class, + "load_class": load_class, + "new_object": new_object, + "get_default_object": get_default_object, + "load_asset": load_asset, + "load_object": load_object, + "find_object": find_object, + } + for name, obj in exported.items(): + setattr(mod, name, obj) + + def configure(**kw): + """Override scripted outcomes, e.g. stub.configure(load_level=False).""" + config.update(**kw) + if not config.level_editor_subsystem: + if hasattr(mod, "LevelEditorSubsystem"): + delattr(mod, "LevelEditorSubsystem") + else: + mod.LevelEditorSubsystem = LevelEditorSubsystem + + mod.configure = configure + configure() # apply presence toggles from overrides + return mod + + +# --------------------------------------------------------------------------- +# Exec harness +# --------------------------------------------------------------------------- + + +@contextmanager +def installed(stub: types.ModuleType): + """Temporarily install ``stub`` as sys.modules['unreal'].""" + prev = sys.modules.get("unreal") + sys.modules["unreal"] = stub + try: + yield + finally: + if prev is None: + sys.modules.pop("unreal", None) + else: + sys.modules["unreal"] = prev + + +def exec_generated(code: str, stub: types.ModuleType, name: str = "") -> str: + """Compile + exec generated UE Python under the stub; return captured stdout. + + Raises SyntaxError if the code does not compile, and propagates any runtime + exception from the generated script (NameError, AttributeError, TypeError...). + """ + code_obj = compile(code, name, "exec") + buf = io.StringIO() + with installed(stub), redirect_stdout(buf): + exec(code_obj, {"__name__": "__ue_generated__"}) # noqa: S102 -- the point of the harness + return buf.getvalue() + + +def parse_result(stdout: str) -> dict: + """Parse captured stdout with the product's RESULT-line grammar. + + Returns {"result": , "output": str, "error": None}. + """ + return _parse_result({"output": stdout, "error": None}) + + +_FAILURE_STRINGS = {"SPAWN_FAILED", "NOT_FOUND", "CREATE_FAILED"} + + +def is_failure(result_data) -> bool: + """Classify a parsed RESULT payload as a failure report. + + dict -> truthy "error" key; str -> known failure markers; list/other -> ok. + """ + if isinstance(result_data, dict): + return bool(result_data.get("error")) + if isinstance(result_data, str): + s = result_data.strip() + return s in _FAILURE_STRINGS or s.startswith("CLASS_NOT_FOUND") + return False + + +def has_result_line(stdout: str) -> bool: + return any(line.startswith("RESULT:") for line in stdout.splitlines()) diff --git a/tests/test_editor.py b/tests/test_editor.py index e28af59..ce217a5 100644 --- a/tests/test_editor.py +++ b/tests/test_editor.py @@ -79,43 +79,6 @@ def test_code_parses(self): ast.parse(code) -class TestUndoRedoCodeGen: - """Generated Python for undo/redo parses cleanly.""" - - def test_undo_code_parses(self): - code = """ -import unreal, json - -try: - result = unreal.EditorLevelLibrary.editor_undo() if hasattr(unreal.EditorLevelLibrary, 'editor_undo') else unreal.SystemLibrary.transaction_undo() - print("RESULT:" + json.dumps({"undone": True})) -except Exception as e: - try: - import unreal - unreal.EditorLevelLibrary.editor_undo() - print("RESULT:" + json.dumps({"undone": True})) - except Exception as e2: - print("RESULT:" + json.dumps({"error": str(e2)})) -""" - ast.parse(code) - - def test_redo_code_parses(self): - code = """ -import unreal, json - -try: - result = unreal.EditorLevelLibrary.editor_redo() if hasattr(unreal.EditorLevelLibrary, 'editor_redo') else unreal.SystemLibrary.transaction_redo() - print("RESULT:" + json.dumps({"redone": True})) -except Exception as e: - try: - unreal.EditorLevelLibrary.editor_redo() - print("RESULT:" + json.dumps({"redone": True})) - except Exception as e2: - print("RESULT:" + json.dumps({"error": str(e2)})) -""" - ast.parse(code) - - class TestFocusActorCodeGen: """Generated Python for ue_focus_actor parses cleanly.""" @@ -209,36 +172,22 @@ async def test_rejects_quit(self, server, mock_ue): class TestUndoAsync: @pytest.mark.asyncio - async def test_happy_path(self, server, mock_ue): + async def test_reports_not_implemented_without_editor_roundtrip(self, server, mock_ue): fn = _call(server, "ue_undo") result = await fn() data = json.loads(result) - assert "error" not in data - mock_ue.execute_python.assert_awaited_once() - - @pytest.mark.asyncio - async def test_code_contains_undo(self, server, mock_ue): - fn = _call(server, "ue_undo") - await fn() - code = mock_ue.execute_python.call_args[0][0] - assert "undo" in code.lower() + assert "not implemented" in data.get("error", "").lower() + mock_ue.execute_python.assert_not_awaited() class TestRedoAsync: @pytest.mark.asyncio - async def test_happy_path(self, server, mock_ue): + async def test_reports_not_implemented_without_editor_roundtrip(self, server, mock_ue): fn = _call(server, "ue_redo") result = await fn() data = json.loads(result) - assert "error" not in data - mock_ue.execute_python.assert_awaited_once() - - @pytest.mark.asyncio - async def test_code_contains_redo(self, server, mock_ue): - fn = _call(server, "ue_redo") - await fn() - code = mock_ue.execute_python.call_args[0][0] - assert "redo" in code.lower() + assert "not implemented" in data.get("error", "").lower() + mock_ue.execute_python.assert_not_awaited() class TestFocusActorAsync: diff --git a/ue_mcp/tools/editor.py b/ue_mcp/tools/editor.py index ca8157a..e44c05f 100644 --- a/ue_mcp/tools/editor.py +++ b/ue_mcp/tools/editor.py @@ -60,9 +60,17 @@ async def console_command(command: str) -> str: return json.dumps(result, indent=2) + # The UE Python API exposes no editor-transaction undo/redo route (verified + # against 5.7; Epic's own 5.8 MCP surface ships none either — see + # docs/EPIC_MCP_MATRIX.md). These previously probed nonexistent APIs and + # errored every call; now they say so up front without an editor round-trip. @server.tool( name="ue_undo", - description="Undo the last editor action. Equivalent to Ctrl+Z.", + description=( + "Undo the last editor action. NOT IMPLEMENTED: no scriptable " + "editor-transaction route exists in the UE Python API — returns an " + "explanatory error. Use Ctrl+Z in the editor." + ), annotations={ "readOnlyHint": False, "destructiveHint": False, @@ -70,34 +78,20 @@ async def console_command(command: str) -> str: }, ) async def undo() -> str: - """Undo the last editor transaction.""" - code = """ -import unreal, json - -success = False -error_msg = "No undo method available" -# editor_undo lives on EditorLevelLibrary; transaction_undo lives on SystemLibrary. -for cls, method_name in [(unreal.EditorLevelLibrary, "editor_undo"), (unreal.SystemLibrary, "transaction_undo")]: - fn = getattr(cls, method_name, None) - if fn is not None: - try: - fn() - success = True - break - except Exception as e: - error_msg = str(e) - -if success: - print("RESULT:" + json.dumps({"undone": True})) -else: - print("RESULT:" + json.dumps({"error": error_msg})) -""" - result = await ue.execute_python(code) - return json.dumps(result, indent=2) + """Honest not-implemented: no verified editor-transaction API exists.""" + return make_error( + "not implemented: the UE Python API exposes no editor-transaction " + "undo route. Use Ctrl+Z in the editor. Tracked for a verified " + "console-exec implementation." + ) @server.tool( name="ue_redo", - description="Redo the last undone editor action. Equivalent to Ctrl+Y.", + description=( + "Redo the last undone editor action. NOT IMPLEMENTED: no scriptable " + "editor-transaction route exists in the UE Python API — returns an " + "explanatory error. Use Ctrl+Y in the editor." + ), annotations={ "readOnlyHint": False, "destructiveHint": False, @@ -105,30 +99,12 @@ async def undo() -> str: }, ) async def redo() -> str: - """Redo the last undone editor transaction.""" - code = """ -import unreal, json - -success = False -error_msg = "No redo method available" -# editor_redo lives on EditorLevelLibrary; transaction_redo lives on SystemLibrary. -for cls, method_name in [(unreal.EditorLevelLibrary, "editor_redo"), (unreal.SystemLibrary, "transaction_redo")]: - fn = getattr(cls, method_name, None) - if fn is not None: - try: - fn() - success = True - break - except Exception as e: - error_msg = str(e) - -if success: - print("RESULT:" + json.dumps({"redone": True})) -else: - print("RESULT:" + json.dumps({"error": error_msg})) -""" - result = await ue.execute_python(code) - return json.dumps(result, indent=2) + """Honest not-implemented: no verified editor-transaction API exists.""" + return make_error( + "not implemented: the UE Python API exposes no editor-transaction " + "redo route. Use Ctrl+Y in the editor. Tracked for a verified " + "console-exec implementation." + ) @server.tool( name="ue_focus_actor", diff --git a/ue_mcp/tools/materials.py b/ue_mcp/tools/materials.py index ed2adf6..a49823d 100644 --- a/ue_mcp/tools/materials.py +++ b/ue_mcp/tools/materials.py @@ -163,53 +163,53 @@ async def get_material_parameters(material_path: str) -> str: return make_error(err) safe_path = escape_for_fstring(material_path) + # Two incompatible API families: get_*_parameter_names wants the base + # Material; get_material_instance_* wants the instance. Branch on which + # we were given, and key with str(name) — unreal.Name keys break json.dumps. code = f""" import unreal, json asset = unreal.EditorAssetLibrary.load_asset("{safe_path}") if asset is None: print("RESULT:" + json.dumps({{"error": "Material not found: {safe_path}"}})) +elif not hasattr(unreal, 'MaterialEditingLibrary'): + print("RESULT:" + json.dumps({{"error": "MaterialEditingLibrary unavailable"}})) else: - params = {{}} - - # Scalar parameters - if hasattr(unreal, 'MaterialEditingLibrary'): - lib = unreal.MaterialEditingLibrary - - try: - scalar_infos = lib.get_scalar_parameter_names(asset) if hasattr(lib, 'get_scalar_parameter_names') else [] - for name in scalar_infos: - try: - val = lib.get_material_instance_scalar_parameter_value(asset, name) - params[name] = {{"type": "scalar", "value": val}} - except Exception: - params[name] = {{"type": "scalar", "value": None}} - except Exception: - pass + lib = unreal.MaterialEditingLibrary + is_instance = hasattr(unreal, 'MaterialInstanceConstant') and isinstance(asset, unreal.MaterialInstanceConstant) + base = asset.get_base_material() if is_instance else asset + if base is None: + print("RESULT:" + json.dumps({{"error": "Material instance has no base material: {safe_path}"}})) + else: + params = {{}} - try: - vector_infos = lib.get_vector_parameter_names(asset) if hasattr(lib, 'get_vector_parameter_names') else [] - for name in vector_infos: + def _collect(names_fn, inst_fn, default_fn, kind, convert): + try: + names = names_fn(base) + except Exception: + return + for name in names: + key = str(name) try: - val = lib.get_material_instance_vector_parameter_value(asset, name) - params[name] = {{"type": "vector", "value": {{"r": val.r, "g": val.g, "b": val.b, "a": val.a}}}} + val = inst_fn(asset, name) if is_instance else default_fn(base, name) + params[key] = {{"type": kind, "value": convert(val)}} except Exception: - params[name] = {{"type": "vector", "value": None}} - except Exception: - pass + params[key] = {{"type": kind, "value": None}} - try: - texture_infos = lib.get_texture_parameter_names(asset) if hasattr(lib, 'get_texture_parameter_names') else [] - for name in texture_infos: - try: - val = lib.get_material_instance_texture_parameter_value(asset, name) - params[name] = {{"type": "texture", "value": val.get_path_name() if val else None}} - except Exception: - params[name] = {{"type": "texture", "value": None}} - except Exception: - pass + _collect(lib.get_scalar_parameter_names, + lib.get_material_instance_scalar_parameter_value, + lib.get_material_default_scalar_parameter_value, + "scalar", lambda v: v) + _collect(lib.get_vector_parameter_names, + lib.get_material_instance_vector_parameter_value, + lib.get_material_default_vector_parameter_value, + "vector", lambda v: {{"r": v.r, "g": v.g, "b": v.b, "a": v.a}} if v is not None else None) + _collect(lib.get_texture_parameter_names, + lib.get_material_instance_texture_parameter_value, + lib.get_material_default_texture_parameter_value, + "texture", lambda v: v.get_path_name() if v else None) - print("RESULT:" + json.dumps({{"material": "{safe_path}", "parameters": params}})) + print("RESULT:" + json.dumps({{"material": "{safe_path}", "is_instance": is_instance, "parameters": params}})) """ result = await ue.execute_python(code) return json.dumps(result, indent=2) From 6085339212ca89116a22a5c63bbf143cd1d224bd Mon Sep 17 00:00:00 2001 From: Joseph Ibrahim Date: Thu, 2 Jul 2026 11:09:33 -0400 Subject: [PATCH 3/4] test(exec-sim): collision-free sentinel values (verify-wave hardening) Adversarial review found the sentinel gate's substring matching maskable: "1.25" hides inside "811.25" (ue_set_transform sx/sy) and "3.25" inside spacing "433.25" (ue_create_cloner z) - a dropped kwarg could pass. All flagged sentinels now use unique fractional patterns. Correction to the prior commit message: registry classification is 47 CODEGEN / 9 DIRECT. Filed for M2 hardening: registry perimeter excludes mcp_server.py-registered tools; DIRECT classification lacks a zero-code assertion; expect_error entries need a contract test; is_failure classifier is whitelist-based. Co-Authored-By: Claude Fable 5 --- tests/exec_sim/registry.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/exec_sim/registry.py b/tests/exec_sim/registry.py index dc081b1..d063648 100644 --- a/tests/exec_sim/registry.py +++ b/tests/exec_sim/registry.py @@ -79,8 +79,10 @@ class ToolEntry: ), ToolEntry( "ue_set_transform", CODEGEN, + # Sentinel values must not be substrings of each other ("1.25" hides + # inside "811.25"), or a dropped kwarg can pass the sentinel gate. kwargs=dict(actor_path=SENTINEL_ACTOR_PATH, x=7317.25, y=811.25, z=97.25, - rx=14.25, ry=28.25, rz=42.25, sx=1.25, sy=2.25, sz=3.25), + rx=14.25, ry=28.25, rz=42.25, sx=51.5625, sy=62.8125, sz=73.1875), sentinel_checkable=("actor_path", "x", "y", "z", "rx", "ry", "rz", "sx", "sy", "sz"), notes="codegen via client; actor-resolver contract gated in test_honesty", ), @@ -141,9 +143,10 @@ class ToolEntry: # ------------------------------------------------------------ mograph.py ToolEntry( "ue_create_cloner", CODEGEN, + # x/y/z use collision-free fractions (3.25 hides inside spacing=433.25). kwargs=dict(layout="Circle", mesh_path=SENTINEL_MESH_PATH, count_x=7317, count_y=6113, count_z=4231, spacing=433.25, - x=1.25, y=2.25, z=3.25, label=SENTINEL_LABEL), + x=151.5625, y=262.8125, z=373.1875, label=SENTINEL_LABEL), sentinel_checkable=("layout", "mesh_path", "count_x", "count_y", "count_z", "spacing", "x", "y", "z", "label"), notes="sentinel gate is the arg-discard trap (layout/counts/spacing/mesh_path)", From 2b70658119aa8ce1ffe4025b5bf667f13175ce86 Mon Sep 17 00:00:00 2001 From: Joseph Ibrahim Date: Thu, 2 Jul 2026 11:17:56 -0400 Subject: [PATCH 4/4] =?UTF-8?q?fix:=20verify-wave=20findings=20=E2=80=94?= =?UTF-8?q?=20stale-frame=20regression,=20cloner=205.7=20reality,=20focus?= =?UTF-8?q?=20route,=20chokepoint=20escaping?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The adversarial verify wave (4 reviewers, one checking against installed UE 5.7 plugin/engine source) failed the branch on two real defects: - perception fallback: a stranded screenshot from a previous capture was returned as the NEXT capture with capture_status:"ok" (empirically reproduced). The trigger pass now removes any pre-existing file before triggering; new honesty contract pins the scenario. - create_cloner could never run on 5.7: the actor reflects as CEClonerActor (/Script/ClonerEffector.CEClonerActor), not the names we probed. Rework per plugin-source facts: correct class resolution; real 5.7 layout catalog (SphereUniform/SphereRandom, no "Sphere"); read-back-verified writes (SetLayoutName silently ignores unknown names, so "applied" now requires a round-trip match); a second configure pass after the async layout load; mesh child attached KEEP_WORLD at the cloner location (was double-transformed). Also from the review notes: - focus_actor: CAMERA ALIGN console fallback (5.7 LevelEditorSubsystem has no focus UFUNCTION) — focus claims now carry the executed route - find_assets escaping moved INTO _CodeGen.find_assets_code so the sync client path is covered too; tool-layer escape removed (double-escape) 563 tests green, ruff clean. Live-editor validation of the cloner second pass lands with the M2 smoke expansion. Co-Authored-By: Claude Fable 5 --- remote_control/codegen.py | 9 +++ tests/exec_sim/test_honesty.py | 66 +++++++++++++++--- ue_mcp/tools/assets.py | 4 +- ue_mcp/tools/editor.py | 17 +++-- ue_mcp/tools/mograph.py | 123 ++++++++++++++++++++++++++------- ue_mcp/tools/perception.py | 10 ++- 6 files changed, 187 insertions(+), 42 deletions(-) diff --git a/remote_control/codegen.py b/remote_control/codegen.py index 380be7b..fe5e32c 100644 --- a/remote_control/codegen.py +++ b/remote_control/codegen.py @@ -5,6 +5,14 @@ """ +def _escape_for_fstring(value: str) -> str: + """Escape backslashes and double quotes for embedding in generated code. + + Escaping lives HERE for find_assets so every caller (MCP tool, sync + client) is covered; other templates still rely on caller-side escaping. + """ + return value.replace("\\", "\\\\").replace('"', '\\"') + class _CodeGen: """Generates UE5 Python scripts. No I/O -- pure string construction.""" @@ -94,6 +102,7 @@ def set_actor_transform_code( @staticmethod def find_assets_code(search_pattern: str, class_filter: str | None = None) -> str: + search_pattern = _escape_for_fstring(search_pattern) return f""" import unreal, json registry = unreal.AssetRegistryHelpers.get_asset_registry() diff --git a/tests/exec_sim/test_honesty.py b/tests/exec_sim/test_honesty.py index 09f26ba..41fe6d5 100644 --- a/tests/exec_sim/test_honesty.py +++ b/tests/exec_sim/test_honesty.py @@ -52,13 +52,20 @@ def test_load_level_failure_is_not_reported_as_loaded(toolbox): # -------------------------------------------------------------------------- -def test_focus_actor_does_not_claim_focus_without_a_focus_api(toolbox): +def test_focus_actor_claims_focus_only_with_an_executed_route(toolbox): + """Without LevelEditorSubsystem (the 5.7 reality), focus must go through the + CAMERA ALIGN console route and SAY SO — a focus claim must always carry the + route that actually executed.""" code = _single_code(toolbox, "ue_focus_actor", actor_label=SENTINEL_LABEL) stub = make_unreal_stub(level_editor_subsystem=False) # no LevelEditorSubsystem at all - result = parse_result(exec_generated(code, stub, name=""))["result"] - assert not (isinstance(result, dict) and result.get("focused")), ( - f"no viewport-focus API exists in this world, yet the code claimed focus: {result!r}" - ) + result = parse_result(exec_generated(code, stub, name=""))["result"] + assert isinstance(result, dict), f"expected dict RESULT, got {result!r}" + if result.get("focused"): + assert result.get("via") == "camera_align_console", ( + f"focus claimed without naming the executed route: {result!r}" + ) + else: + assert result.get("error"), f"neither an honest focus nor an honest error: {result!r}" # -------------------------------------------------------------------------- @@ -257,10 +264,49 @@ async def execute_python(self, code: str) -> dict: "fallback regressed to a single editor execution — the screenshot file " "can never exist in the same exec that triggered it" ) + assert result_is_honest_miss(final) + + +def result_is_honest_miss(final: dict) -> bool: result = final.get("result") - assert isinstance(result, dict), f"expected dict result, got {final!r}" - assert result.get("image") == "", "no screenshot existed; image must be empty" - assert result.get("capture_status") in {"timeout", "trigger_failed"}, ( - "empty image must be flagged via capture_status, got " - f"{result.get('capture_status')!r} in {({k: v for k, v in result.items() if k != 'image'})!r}" + return ( + isinstance(result, dict) + and result.get("image") == "" + and result.get("capture_status") in {"timeout", "trigger_failed"} ) + + +def test_viewport_fallback_does_not_return_a_stale_frame_as_ok(monkeypatch): + """A stranded screenshot from a PREVIOUS capture (timeout/read-failure) must + not be returned as this capture's frame — the trigger pass removes it before + triggering. (Empirically reproduced regression from the verify wave.)""" + from ue_mcp.tools import perception + + monkeypatch.setattr(perception, "FALLBACK_POLL_ATTEMPTS", 3) + monkeypatch.setattr(perception, "FALLBACK_POLL_INTERVAL_S", 0.0) + + out_path = os.path.join(tempfile.gettempdir(), "ue_perception_capture.jpeg") + with open(out_path, "wb") as f: + f.write(b"STALE-FRAME-FROM-PREVIOUS-SESSION") + try: + stub = make_unreal_stub(screenshot_writes_file=False) # this capture never lands + + class EditorSim: + def __init__(self): + self.scripts: list[str] = [] + + async def execute_python(self, code: str) -> dict: + self.scripts.append(code) + return parse_result(exec_generated(code, stub, name=f"")) + + final = asyncio.run(perception._fallback_capture(EditorSim(), 320, 200, "jpeg")) + result = final.get("result") + assert isinstance(result, dict), f"expected dict result, got {final!r}" + assert result.get("capture_status") != "ok", ( + f"a stale pre-existing file was returned as this capture's frame: " + f"{({k: v for k, v in result.items() if k != 'image'})!r}" + ) + assert "STALE" not in (result.get("image") or ""), "stale bytes leaked into the payload" + finally: + if os.path.exists(out_path): + os.remove(out_path) diff --git a/ue_mcp/tools/assets.py b/ue_mcp/tools/assets.py index 08a079e..73e7225 100644 --- a/ue_mcp/tools/assets.py +++ b/ue_mcp/tools/assets.py @@ -35,7 +35,9 @@ async def find_assets(search_pattern: str, class_filter: str | None = None) -> s if err := sanitize_class_name(class_filter, "class_filter"): return make_error(err) - result = await ue.find_assets(escape_for_fstring(search_pattern), class_filter=class_filter) + # Escaping happens inside _CodeGen.find_assets_code (the chokepoint), + # covering the sync client too — do not escape here or it doubles. + result = await ue.find_assets(search_pattern, class_filter=class_filter) return json.dumps(result, indent=2) @server.tool( diff --git a/ue_mcp/tools/editor.py b/ue_mcp/tools/editor.py index e44c05f..2f04225 100644 --- a/ue_mcp/tools/editor.py +++ b/ue_mcp/tools/editor.py @@ -130,14 +130,23 @@ async def focus_actor(actor_label: str) -> str: print("RESULT:" + json.dumps({{"error": "Actor not found: {safe_label}"}})) else: subsystem.set_selected_level_actors([actor]) - focused = False + focused_via = None if hasattr(unreal, 'LevelEditorSubsystem'): le_sub = unreal.get_editor_subsystem(unreal.LevelEditorSubsystem) if hasattr(le_sub, 'focus_on_selected_actors'): le_sub.focus_on_selected_actors() - focused = True - if focused: - print("RESULT:" + json.dumps({{"focused": "{safe_label}"}})) + focused_via = "level_editor_subsystem" + if focused_via is None: + # 5.7's LevelEditorSubsystem exposes no focus UFUNCTION; the editor + # console command aligns the active viewport to the selection instead. + try: + world = unreal.EditorLevelLibrary.get_editor_world() + unreal.SystemLibrary.execute_console_command(world, "CAMERA ALIGN ACTIVEVIEWPORT") + focused_via = "camera_align_console" + except Exception: + pass + if focused_via: + print("RESULT:" + json.dumps({{"focused": "{safe_label}", "via": focused_via}})) else: print("RESULT:" + json.dumps({{"error": "No viewport focus method available", "selected": "{safe_label}"}})) """ diff --git a/ue_mcp/tools/mograph.py b/ue_mcp/tools/mograph.py index 48d91fd..f9aab7a 100644 --- a/ue_mcp/tools/mograph.py +++ b/ue_mcp/tools/mograph.py @@ -6,6 +6,7 @@ from __future__ import annotations +import asyncio import json import logging @@ -14,6 +15,10 @@ logger = logging.getLogger("ue5-mcp.tools.mograph") +# UCEClonerComponent.SetClonerActiveLayout loads the layout instance +# asynchronously — a second editor pass configures it after this delay. +CLONER_LAYOUT_POLL_DELAY_S = 0.5 + def register(server: MCPServer, ue: UEBridge) -> None: @server.tool( @@ -40,8 +45,11 @@ async def create_cloner( z: float = 0.0, label: str | None = None, ) -> str: - """Create a ClonerEffector. layout can be: Grid, Circle, Line, Sphere, Honeycomb, Cylinder.""" - valid_layouts = {"Grid", "Circle", "Line", "Sphere", "Honeycomb", "Cylinder"} + """Create a ClonerEffector. layout: Grid, Line, Circle, SphereUniform, + SphereRandom, Honeycomb, Cylinder (5.7 registered layout names). + count_x/y/z and spacing map fully onto the Grid layout; other layouts + apply what their property set supports and report the rest skipped.""" + valid_layouts = {"Grid", "Line", "Circle", "SphereUniform", "SphereRandom", "Honeycomb", "Cylinder"} if layout not in valid_layouts: return make_error(f"Invalid layout '{layout}'. Must be one of: {', '.join(sorted(valid_layouts))}") if err := sanitize_content_path(mesh_path, "mesh_path"): @@ -52,19 +60,20 @@ async def create_cloner( label_str = escape_for_fstring(label or "ClaudeCloner") safe_mesh = escape_for_fstring(mesh_path) - # ClonerEffector clones its ATTACHED child actors; layout/count/spacing - # live on the cloner component and its active layout object. Property - # names could not be verified against a live editor, so every write goes - # through _safe_set and is reported applied/skipped (lighting.py pattern) - # instead of silently pretending. + # Verified against the installed 5.7 ClonerEffector plugin source: + # - the actor reflects as CEClonerActor (/Script/ClonerEffector.CEClonerActor) + # - SetLayoutName silently ignores unknown names, so every write is + # verified by read-back before it may be reported "applied" + # - the active layout loads asynchronously after layout_name is set, + # so count/spacing usually need the second pass below code = f""" import unreal, json subsystem = unreal.get_editor_subsystem(unreal.EditorActorSubsystem) -cloner_class = unreal.find_class("ClonerActor") or unreal.find_class("ACEClonerActor") +cloner_class = unreal.find_class("CEClonerActor") if cloner_class is None: - cloner_class = unreal.load_class(None, "/Script/ClonerEffector.ClonerActor") + cloner_class = unreal.load_class(None, "/Script/ClonerEffector.CEClonerActor") if cloner_class is None: print("RESULT:" + json.dumps({{"error": "CLASS_NOT_FOUND - ClonerEffector plugin may not be loaded"}})) @@ -80,45 +89,47 @@ async def create_cloner( cloner.set_actor_label("{label_str}") applied, skipped = [], [] - def _safe_set(obj, name, value): + def _verified_set(obj, name, value): try: obj.set_editor_property(name, value) - applied.append(name) + back = obj.get_editor_property(name) + if str(back) == str(value): + applied.append(name) + else: + skipped.append(name + ": write ignored (read-back " + str(back)[:40] + ")") except Exception as e: skipped.append(name + ": " + str(e)[:80]) - comp = None + layout_pending = False comp_class = getattr(unreal, "CEClonerComponent", None) - if comp_class: - comp = cloner.get_component_by_class(comp_class) + comp = cloner.get_component_by_class(comp_class) if comp_class else None if comp is None: skipped.append("layout/count/spacing: CEClonerComponent not found on actor") else: - _safe_set(comp, "layout_name", "{layout}") + _verified_set(comp, "layout_name", "{layout}") layout_obj = None try: layout_obj = comp.get_editor_property("active_layout") except Exception: pass - target = layout_obj if layout_obj is not None else comp - _safe_set(target, "count_x", {count_x}) - _safe_set(target, "count_y", {count_y}) - _safe_set(target, "count_z", {count_z}) - _safe_set(target, "spacing_x", {spacing}) - _safe_set(target, "spacing_y", {spacing}) - _safe_set(target, "spacing_z", {spacing}) + if layout_obj is not None and "{layout}".lower() in type(layout_obj).__name__.lower(): + for prop, value in [("count_x", {count_x}), ("count_y", {count_y}), ("count_z", {count_z}), + ("spacing_x", {spacing}), ("spacing_y", {spacing}), ("spacing_z", {spacing})]: + _verified_set(layout_obj, prop, value) + else: + layout_pending = True mesh_attached = False try: mesh = unreal.EditorAssetLibrary.load_asset("{safe_mesh}") if mesh: child = subsystem.spawn_actor_from_class( - unreal.StaticMeshActor, unreal.Vector({x}, {y}, {z}), unreal.Rotator(0, 0, 0)) + unreal.StaticMeshActor, cloner.get_actor_location(), unreal.Rotator(0, 0, 0)) if child: child.static_mesh_component.set_static_mesh(mesh) - child.attach_to_actor(cloner, "", unreal.AttachmentRule.KEEP_RELATIVE, - unreal.AttachmentRule.KEEP_RELATIVE, - unreal.AttachmentRule.KEEP_RELATIVE, False) + child.attach_to_actor(cloner, "", unreal.AttachmentRule.KEEP_WORLD, + unreal.AttachmentRule.KEEP_WORLD, + unreal.AttachmentRule.KEEP_WORLD, False) mesh_attached = True else: skipped.append("mesh: asset not found {safe_mesh}") @@ -128,12 +139,72 @@ def _safe_set(obj, name, value): print("RESULT:" + json.dumps({{ "created": cloner.get_path_name(), "layout": "{layout}", + "layout_pending": layout_pending, "applied": applied, "skipped": skipped, "mesh_attached": mesh_attached, }})) """ result = await ue.execute_python(code) + first = result.get("result") if isinstance(result, dict) else None + if not isinstance(first, dict) or not first.get("layout_pending"): + return json.dumps(result, indent=2) + + # Layout instance was still loading — configure it in a second pass. + await asyncio.sleep(CLONER_LAYOUT_POLL_DELAY_S) + cloner_path = escape_for_fstring(str(first.get("created", ""))) + code2 = f""" +import unreal, json + +applied, skipped = [], [] +layout_pending = True +layout_class = None + +def _verified_set(obj, name, value): + try: + obj.set_editor_property(name, value) + back = obj.get_editor_property(name) + if str(back) == str(value): + applied.append(name) + else: + skipped.append(name + ": write ignored (read-back " + str(back)[:40] + ")") + except Exception as e: + skipped.append(name + ": " + str(e)[:80]) + +cloner = unreal.load_object(None, "{cloner_path}") +comp_class = getattr(unreal, "CEClonerComponent", None) +comp = cloner.get_component_by_class(comp_class) if (cloner and comp_class) else None +if comp is None: + skipped.append("configure: cloner or CEClonerComponent not found") +else: + layout_obj = None + try: + layout_obj = comp.get_editor_property("active_layout") + except Exception: + pass + if layout_obj is not None: + layout_class = type(layout_obj).__name__ + if layout_obj is not None and "{layout}".lower() in type(layout_obj).__name__.lower(): + layout_pending = False + for prop, value in [("count_x", {count_x}), ("count_y", {count_y}), ("count_z", {count_z}), + ("spacing_x", {spacing}), ("spacing_y", {spacing}), ("spacing_z", {spacing})]: + _verified_set(layout_obj, prop, value) + else: + skipped.append("layout not active after wait (" + str(layout_class) + ") - counts/spacing not applied") + +print("RESULT:" + json.dumps({{"applied": applied, "skipped": skipped, + "layout_pending": layout_pending, "layout_class": layout_class}})) +""" + result2 = await ue.execute_python(code2) + second = result2.get("result") if isinstance(result2, dict) else None + if isinstance(second, dict): + first["applied"] = list(first.get("applied", [])) + list(second.get("applied", [])) + first["skipped"] = list(first.get("skipped", [])) + list(second.get("skipped", [])) + first["layout_pending"] = bool(second.get("layout_pending")) + first["layout_class"] = second.get("layout_class") + else: + first["skipped"] = list(first.get("skipped", [])) + [ + "configure pass failed: " + str((result2 or {}).get("error"))[:80]] return json.dumps(result, indent=2) @server.tool( diff --git a/ue_mcp/tools/perception.py b/ue_mcp/tools/perception.py index 81d82f1..82f72d5 100644 --- a/ue_mcp/tools/perception.py +++ b/ue_mcp/tools/perception.py @@ -104,7 +104,7 @@ async def _fallback_capture(ue, width: int, height: int, format: str) -> dict: a missing file and (before the fix) reported success with an empty image. """ trigger_code = f""" -import unreal, json, tempfile +import unreal, json, tempfile, os world = unreal.EditorLevelLibrary.get_editor_world() subsystem = unreal.get_editor_subsystem(unreal.EditorActorSubsystem) @@ -128,6 +128,14 @@ async def _fallback_capture(ue, width: int, height: int, format: str) -> dict: tmp_dir = tempfile.gettempdir().replace("\\\\", "/") out_path = tmp_dir + "/ue_perception_capture.{format}" +# A stranded file from a previous capture (timeout/read-failure) would be +# returned as THIS capture's frame — remove it before triggering. +try: + if os.path.exists(out_path): + os.remove(out_path) +except Exception: + pass + trigger = "none" try: unreal.AutomationLibrary.take_high_res_screenshot({width}, {height}, out_path)