From 8b56df3d50e37020454cb5061527d50c59140501 Mon Sep 17 00:00:00 2001 From: Clay Date: Fri, 11 Sep 2026 15:19:54 -0400 Subject: [PATCH] fix: register runtime actors for depth and segmentation --- .github/workflows/python-regressions.yml | 37 +++++ docs/source/components/ue_detail.rst | 47 +++++++ examples/README.md | 3 + examples/verify_runtime_sensors.py | 163 +++++++++++++++++++++++ simworld/communicator/communicator.py | 11 +- simworld/communicator/unrealcv.py | 32 ++++- tests/requirements.txt | 13 ++ tests/test_runtime_spawn_sensors.py | 121 +++++++++++++++++ 8 files changed, 420 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/python-regressions.yml create mode 100644 examples/verify_runtime_sensors.py create mode 100644 tests/requirements.txt create mode 100644 tests/test_runtime_spawn_sensors.py diff --git a/.github/workflows/python-regressions.yml b/.github/workflows/python-regressions.yml new file mode 100644 index 00000000..d2582275 --- /dev/null +++ b/.github/workflows/python-regressions.yml @@ -0,0 +1,37 @@ +name: Python regressions + +on: [push, pull_request] + +permissions: + contents: read + +jobs: + tests: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest] + python: ['3.10', '3.12'] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python }} + cache: pip + cache-dependency-path: tests/requirements.txt + - name: Install regression test dependencies + run: python -m pip install -r tests/requirements.txt + - name: Run regression tests + env: + PYTEST_DISABLE_PLUGIN_AUTOLOAD: '1' + run: python -m pytest tests -q --junitxml=regressions.xml --basetemp=test-output + - name: Upload test evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: regressions-${{ matrix.os }}-python-${{ matrix.python }} + path: | + regressions.xml + test-output/**/*.mp4 + test-output/**/*.png diff --git a/docs/source/components/ue_detail.rst b/docs/source/components/ue_detail.rst index efb0fe56..79499666 100644 --- a/docs/source/components/ue_detail.rst +++ b/docs/source/components/ue_detail.rst @@ -55,6 +55,53 @@ Sensors As illustrated in the figure above, SimWorld supports a variety of sensors, including RGB images, segmentation maps, and depth images, enabling a rich understanding of the surrounding environment. +Runtime actors in depth and segmentation +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The ``Communicator`` methods for spawning objects, agents, scooters, vehicles, +pedestrians, traffic signals, and waypoint marks register their render components +for ``depth`` and ``object_mask`` after setting the initial transform and mobility. +In the Base20260201 Windows backend, creating an actor alone makes it visible in +``lit``, but the color annotation command is also required for these two sensors. +No Blueprint modification is needed for the bundled box, humanoid, and vehicle +assets. + +Runtime labels default to a stable, nonblack RGB color derived from the actor +name. These colors are not guaranteed to be globally unique or to match a semantic +class palette. Use ``ucv.set_color(name, (r, g, b))`` for a dataset's explicit +labels. This changes the sensor annotation, not the actor's visible material. +Procedurally generated city assets keep their configured asset-library colors. + +For the low-level ``spawn_bp_asset`` API, register explicitly after configuration: + +.. code-block:: python + + name = 'my_box' + ucv.spawn_bp_asset('/Game/CityDatabase/blueprints/BP_Box.BP_Box_C', name) + ucv.set_location((0, 0, 150), name) + ucv.set_scale((1, 1, 1), name) + ucv.set_movable(name, True) + ucv.set_color(name, (251, 13, 107)) + +Keep moving actors movable so their annotation follows subsequent transforms. +The spawn and annotation methods raise ``RuntimeError`` if the server does not +acknowledge the operation, including when an asset path is unavailable. + +To verify the behavior on a dedicated server running ``/Game/Maps/empty`` with +the complete Base20260201 asset package, set its UnrealCV port to 19090 and run +from the repository root: + +.. code-block:: console + + python -m examples.verify_runtime_sensors --port 19090 --output sensor-evidence + +The script exercises ``spawn_object``, ``spawn_agent``, and ``spawn_vehicles`` and +then moves the actors. It writes unmodified RGB/mask PNGs, metric depth NPYs, +the request transcript, and per-actor statistics to ``sensor-evidence``. It exits +with an error if an actor is missing from its mask, lacks foreground depth, or +its mask fails to move. It removes only the actors and capture camera it creates. +The camera is placed away from the player pawn to avoid capturing its own body. + How to get images ~~~~~~~~~~~~~~~~~ diff --git a/examples/README.md b/examples/README.md index e09ec994..d7c386b9 100644 --- a/examples/README.md +++ b/examples/README.md @@ -4,6 +4,9 @@ This folder contains example code demonstrating various functionalities of SimWo ## Example List +### `verify_runtime_sensors.py` +Verify runtime boxes, humanoids, and vehicles in depth and segmentation on a dedicated Base20260201 `/Game/Maps/empty` server. Run `python -m examples.verify_runtime_sensors --port 19090 --output sensor-evidence` from the repository root. The script saves raw PNG/NPY captures, a command transcript, and measurements before and after moving the actors; it exits nonzero on failure and removes its test actors and camera. + ### `gym_interface_demo.ipynb` Minimal demo showing how to create an LLM-based agent with a Gym-like environment interface. Demonstrates Agent-Environment interaction loop and goal-oriented tasks. diff --git a/examples/verify_runtime_sensors.py b/examples/verify_runtime_sensors.py new file mode 100644 index 00000000..249c9259 --- /dev/null +++ b/examples/verify_runtime_sensors.py @@ -0,0 +1,163 @@ +"""Verify runtime actor sensors on a dedicated Base20260201 empty-map server. + +Run from the repository root with ``python -m examples.verify_runtime_sensors +--port 19090 --output sensor-evidence``. Saves raw captures and exits nonzero if +boxes, humanoids, or vehicles are missing from depth or object_mask after spawning +or moving. The script creates a camera and three actors, then removes them. +""" + +import argparse +import hashlib +import json +import re +import time +from io import BytesIO +from pathlib import Path +from types import SimpleNamespace + +import numpy as np +from PIL import Image + +from simworld.communicator.communicator import Communicator +from simworld.communicator.unrealcv import UnrealCV +from simworld.utils.vector import Vector + + +def capture(request, camera, output): + """Save the server's original PNG/NPY bytes without client visualization.""" + output.mkdir(parents=True, exist_ok=True) + images = {} + for mode, extension in [('lit', 'png'), ('depth', 'npy'), ('object_mask', 'png')]: + payload = request(f'vget /camera/{camera}/{mode} {extension}') + if not isinstance(payload, bytes): + raise RuntimeError(f'{mode} capture failed: {payload!r}') + (output / f'{mode}.{extension}').write_bytes(payload) + if extension == 'npy': + images[mode] = np.load(BytesIO(payload), allow_pickle=False) + else: + images[mode] = np.asarray(Image.open(BytesIO(payload)).convert('RGB')) + return images + + +def measure(images, background, colors): + """Measure each actor's mask area and foreground depth at those same pixels.""" + results = {} + for name, color in colors.items(): + region = np.all(images['object_mask'] == color, axis=-1) + ys, xs = np.nonzero(region) + count = int(region.sum()) + closer = images['depth'][region] < background['depth'][region] - 1.0 + results[name] = { + 'rgb': color, 'mask_pixels': count, + 'centroid_x': float(xs.mean()) if count else None, + 'centroid_y': float(ys.mean()) if count else None, + 'foreground_depth_fraction': float(closer.mean()) if count else 0.0, + 'median_depth_cm': float(np.median(images['depth'][region])) if count else None, + 'median_background_depth_cm': float(np.median(background['depth'][region])) if count else None, + 'passed': count > 50 and float(closer.mean()) > 0.9, + } + return results + + +def main(): + """Exercise public spawn methods, capture evidence, and check moving actors.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--port', type=int, required=True, help='Port of a dedicated empty-map server') + parser.add_argument('--output', type=Path, required=True) + args = parser.parse_args() + args.output.mkdir(parents=True, exist_ok=True) + ucv = UnrealCV(port=args.port, resolution=(640, 480)) + communicator = Communicator(ucv) + original_request = ucv.client.request + transcript = [] + + def request(command, *positional, **kwargs): + """Record the real protocol exchange, hashing binary responses.""" + if not positional: + kwargs.setdefault('timeout', 30) + response = original_request(command, *positional, **kwargs) + record = {'command': command} + if isinstance(response, bytes): + record.update(bytes=len(response), sha256=hashlib.sha256(response).hexdigest()) + else: + record['response'] = response + transcript.append(record) + return response + + ucv.client.request = request + box = 'SensorCheckBox' + human = SimpleNamespace(id='sensor_check', position=Vector(0, 0), direction=Vector(-1, 0)) + vehicle = SimpleNamespace(id='sensor_check', position=Vector(0, 450), direction=Vector(1, 0), + vehicle_reference='/Game/TrafficSystem/Vehicle/Vehicle1.Vehicle1_C') + names = [box, communicator.get_humanoid_name(human.id), communicator.get_vehicle_name(vehicle.id)] + created = [] + camera_actor = None + summary = {} + try: + existing = ucv.get_objects() + if any(name in existing for name in names): + raise RuntimeError('SensorCheck actors already exist; use a fresh dedicated empty-map server') + (args.output / 'status.txt').write_text(str(request('vget /unrealcv/status')), encoding='utf-8') + cameras_before = ucv.get_cameras().split() + camera_actor = request('vset /cameras/spawn') + if len(ucv.get_cameras().split()) != len(cameras_before) + 1: + raise RuntimeError('Expected one new capture camera') + # The endpoint lists sensor names, but capture commands take their indices. + camera = len(cameras_before) + ucv.set_camera_location(camera, (-1200, 0, 500)) + ucv.set_camera_rotation(camera, (-16, 0, 0)) + ucv.set_camera_resolution(camera, (640, 480)) + ucv.set_camera_fov(camera, 90) + time.sleep(1) + background = capture(request, camera, args.output / 'background') + + created.append(box) + communicator.spawn_object(box, '/Game/CityDatabase/blueprints/BP_Box.BP_Box_C', + (0, -350, 150), (0, 0, 0)) + ucv.set_scale((3, 3, 3), box) + created.append(names[1]) + communicator.spawn_agent(human, None, position=(0, 0, 600)) + created.append(names[2]) + communicator.spawn_vehicles([vehicle]) + time.sleep(4) + colors = {} + for name in names: + response = str(request(f'vget /object/{name}/color')) + values = re.search(r'R=(\d+),G=(\d+),B=(\d+)', response) + if values is None: + raise RuntimeError(f'Cannot read {name} label: {response}') + colors[name] = [int(value) for value in values.groups()] + spawned = capture(request, camera, args.output / 'spawned') + summary['spawned'] = measure(spawned, background, colors) + summary['spawned_depth_identical_to_background'] = bool(np.array_equal(spawned['depth'], background['depth'])) + summary['spawned_mask_identical_to_background'] = bool(np.array_equal(spawned['object_mask'], background['object_mask'])) + + for name, location in zip(names, [(0, -500, 150), (0, -150, 150), (0, 650, 150)]): + ucv.set_location(location, name) + time.sleep(2) + moved = capture(request, camera, args.output / 'moved') + summary['moved'] = measure(moved, background, colors) + for name in names: + before_x = summary['spawned'][name]['centroid_x'] + after_x = summary['moved'][name]['centroid_x'] + shift = abs(after_x - before_x) if before_x is not None and after_x is not None else 0.0 + summary['moved'][name]['centroid_shift_pixels'] = shift + summary['moved'][name]['passed'] &= shift > 5 + summary['passed'] = all(result['passed'] for stage in ('spawned', 'moved') for result in summary[stage].values()) + (args.output / 'summary.json').write_text(json.dumps(summary, indent=2), encoding='utf-8') + print(json.dumps(summary, indent=2)) + if not summary['passed']: + raise RuntimeError('Runtime sensor verification failed; see summary.json and raw captures') + finally: + try: + existing = ucv.get_objects() + for name in created + ([camera_actor] if camera_actor else []): + if name in existing: + ucv.destroy(name) + finally: + (args.output / 'transcript.json').write_text(json.dumps(transcript, indent=2), encoding='utf-8') + ucv.disconnect() + + +if __name__ == '__main__': + main() diff --git a/simworld/communicator/communicator.py b/simworld/communicator/communicator.py index ab9a8b1d..d79785f3 100644 --- a/simworld/communicator/communicator.py +++ b/simworld/communicator/communicator.py @@ -598,6 +598,7 @@ def spawn_object(self, object_name, model_path, position, direction): self.unrealcv.set_scale((1, 1, 1), object_name) self.unrealcv.set_collision(object_name, True) self.unrealcv.set_movable(object_name, True) + self.unrealcv.set_color(object_name) # Initialization methods def spawn_agent(self, agent, name, position=None, model_path='/Game/TrafficSystem/Pedestrian/Base_User_Agent.Base_User_Agent_C', type='humanoid'): @@ -643,6 +644,7 @@ def spawn_agent(self, agent, name, position=None, model_path='/Game/TrafficSyste self.unrealcv.set_scale((1, 1, 1), name) # Default scale self.unrealcv.set_collision(name, True) self.unrealcv.set_movable(name, True) + self.unrealcv.set_color(name) def spawn_scooter(self, scooter, model_path): """Spawn scooter. @@ -670,6 +672,7 @@ def spawn_scooter(self, scooter, model_path): self.unrealcv.set_scale((1, 1, 1), name) # Default scale self.unrealcv.set_collision(name, True) self.unrealcv.set_movable(name, True) + self.unrealcv.set_color(name) def spawn_vehicles(self, vehicles): """Spawn vehicles. @@ -697,6 +700,7 @@ def spawn_vehicles(self, vehicles): self.unrealcv.set_scale((1, 1, 1), name) # Default scale self.unrealcv.set_collision(name, True) self.unrealcv.set_movable(name, True) + self.unrealcv.set_color(name) def spawn_pedestrians(self, pedestrians, model_path='/Game/TrafficSystem/Pedestrian/Base_Pedestrian.Base_Pedestrian_C'): """Spawn pedestrians. @@ -725,6 +729,7 @@ def spawn_pedestrians(self, pedestrians, model_path='/Game/TrafficSystem/Pedestr self.unrealcv.set_scale((1, 1, 1), name) # Default scale self.unrealcv.set_collision(name, True) self.unrealcv.set_movable(name, True) + self.unrealcv.set_color(name) def spawn_traffic_signals(self, traffic_signals, traffic_light_model_path='/Game/city_props/BP/props/street_light/BP_street_light.BP_street_light_C', pedestrian_light_model_path='/Game/city_props/BP/props/street_light/BP_street_light_ped.BP_street_light_ped_C'): """Spawn traffic signals. @@ -758,6 +763,7 @@ def spawn_traffic_signals(self, traffic_signals, traffic_light_model_path='/Game self.unrealcv.set_scale((1, 1, 1), name) # Default scale self.unrealcv.set_collision(name, True) self.unrealcv.set_movable(name, False) + self.unrealcv.set_color(name) def spawn_intersection(self, intersection_name, model_path): """Spawn intersection. @@ -795,6 +801,7 @@ def spawn_waypoint_mark(self, waypoints, model_path): self.unrealcv.set_scale((1, 1, 1), name) self.unrealcv.set_collision(name, False) self.unrealcv.set_movable(name, False) + self.unrealcv.set_color(name) def spawn_ue_manager(self, ue_manager_path): """Spawn UE manager. @@ -860,8 +867,6 @@ def _process_node(row): return else: self.unrealcv.spawn_bp_asset(instance_ref, id) - if run_time: - self.unrealcv.set_color(id, rgb_values) location = node_df.loc[id, ['properties_location_x', 'properties_location_y', 'properties_location_z']].to_list() self.unrealcv.set_location(location, id) orientation = node_df.loc[id, ['properties_orientation_pitch', 'properties_orientation_yaw', 'properties_orientation_roll']].to_list() @@ -870,6 +875,8 @@ def _process_node(row): self.unrealcv.set_scale(scale, id) self.unrealcv.set_collision(id, True) self.unrealcv.set_movable(id, False) + if run_time: + self.unrealcv.set_color(id, rgb_values) generated_ids.add(id) node_df.apply(_process_node, axis=1) diff --git a/simworld/communicator/unrealcv.py b/simworld/communicator/unrealcv.py index 77801502..f60fcb8a 100644 --- a/simworld/communicator/unrealcv.py +++ b/simworld/communicator/unrealcv.py @@ -4,6 +4,7 @@ allowing for various operations such as object spawning, movement, and image capture. """ +import hashlib import json import os import struct @@ -91,10 +92,15 @@ def spawn_bp_asset(self, prefab_path, name): Args: prefab_path: Prefab path. name: Object name. + + Raises: + RuntimeError: The server rejects the asset or does not acknowledge it. """ cmd = f'vset /objects/spawn_bp_asset {prefab_path} {name}' with self.lock: - self.client.request(cmd) + response = self.client.request(cmd) + if response not in ('ok', str(name)): + raise RuntimeError(f'Failed to spawn {name!r} from {prefab_path!r}: {response!r}') def clean_garbage(self): """Clean garbage objects.""" @@ -137,17 +143,33 @@ def set_scale(self, scale, name): with self.lock: self.client.request(cmd) - def set_color(self, actor_name, color): - """Set object color. + def set_color(self, actor_name, color=None): + """Register an object's render components and set its sensor label color. + + The packaged backend needs this annotation for both depth and object_mask. + Call after setting the initial transform and component mobility. This does + not change the object's visible material in the lit image. + + Omitted colors use a stable, nonblack label derived from the actor name. + Supply an explicit color for a dataset's semantic or instance palette. Args: actor_name: Object name. - color: Color in the form [R, G, B]. + color: Label in the form [R, G, B], or None for an automatic label. + + Raises: + RuntimeError: The server does not acknowledge sensor registration. """ + if color is None: + color = tuple(hashlib.sha256(str(actor_name).encode('utf-8')).digest()[:3]) + if color == (0, 0, 0): + color = (1, 1, 1) [R, G, B] = color cmd = f'vset /object/{actor_name}/color {R} {G} {B}' with self.lock: - self.client.request(cmd) + response = self.client.request(cmd) + if response != 'ok': + raise RuntimeError(f'Failed to register sensor color for {actor_name!r}: {response!r}') def enable_controller(self, name, enable_controller): """Enable or disable controller. diff --git a/tests/requirements.txt b/tests/requirements.txt new file mode 100644 index 00000000..90499606 --- /dev/null +++ b/tests/requirements.txt @@ -0,0 +1,13 @@ +# Dependencies needed by the regression tests and simworld's package imports. +# Asset retrieval models are not exercised by this suite. +pytest +numpy +pandas +pyqtgraph +PyQt5 +unrealcv +opencv-python +Pillow +openai +PyYAML +ipython diff --git a/tests/test_runtime_spawn_sensors.py b/tests/test_runtime_spawn_sensors.py new file mode 100644 index 00000000..3e4e20fc --- /dev/null +++ b/tests/test_runtime_spawn_sensors.py @@ -0,0 +1,121 @@ +"""Regression coverage for runtime actor sensor registration and spawn failures.""" + +import json +from threading import Lock +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest + +from simworld.communicator.communicator import Communicator +from simworld.communicator.unrealcv import UnrealCV +from simworld.utils.vector import Vector + + +@pytest.fixture +def client(): + """Exercise the production protocol methods without requiring Unreal Engine.""" + client = UnrealCV.__new__(UnrealCV) + client.lock = Lock() + client.client = Mock() + client.client.request.return_value = 'ok' + return client + + +def commands(client): + """Return commands observed at the network boundary.""" + return [call.args[0] for call in client.client.request.call_args_list] + + +@pytest.mark.parametrize('kind', ['object', 'agent', 'scooter', 'vehicles', 'pedestrians', 'traffic_signals', 'waypoint_mark']) +def test_runtime_spawners_register_after_placement_and_mobility(client, kind): + """All rendered runtime actors need annotation at their final initial pose.""" + communicator = Communicator(client) + actor = SimpleNamespace(id=7, position=Vector(150, 250), direction=Vector(0, 1), + vehicle_reference='/Game/Vehicle.Vehicle_C', type='both') + model = '/Game/Actor.Actor_C' + if kind == 'object': + communicator.spawn_object('Box', model, (150, 250, 100), (0, 90, 0)) + elif kind == 'agent': + communicator.spawn_agent(actor, None, position=(150, 250, 100), model_path=model) + elif kind == 'scooter': + communicator.spawn_scooter(actor, model) + elif kind in ('vehicles', 'pedestrians', 'traffic_signals'): + getattr(communicator, f'spawn_{kind}')([actor]) + else: + communicator.spawn_waypoint_mark([actor], model) + + sent = commands(client) + name = sent[0].split()[-1] + labels = [command for command in sent if command.startswith(f'vset /object/{name}/color ')] + assert len(labels) == 1, f'{kind} did not register the actor for depth/object_mask' + rgb = tuple(map(int, labels[0].split()[-3:])) + assert all(0 <= value <= 255 for value in rgb) + assert rgb != (0, 0, 0) + for setting in ('location', 'rotation', 'scale', 'collision', 'object_mobility'): + placement = next(command for command in sent if command.startswith(f'vset /object/{name}/{setting} ')) + assert sent.index(placement) < sent.index(labels[0]) + + +def test_default_labels_are_stable_and_actor_specific(client): + """A repeated actor name retains its label without Python's randomized hash.""" + client.set_color('Box') + client.set_color('Pedestrian') + client.set_color('Box') + colors = [command.split()[-3:] for command in commands(client)] + assert colors[0] == colors[2] + assert colors[0] != colors[1] + + +def test_explicit_label_is_preserved(client): + """Existing semantic palettes must still send the exact requested RGB value.""" + client.set_color('Road', (12, 34, 56)) + assert commands(client) == ['vset /object/Road/color 12 34 56'] + + +@pytest.mark.parametrize('response', ['error Can not find object', None]) +def test_registration_failure_is_reported(client, response): + """A missing annotation must not silently look like a successful spawn.""" + client.client.request.return_value = response + with pytest.raises(RuntimeError, match='Road'): + client.set_color('Road', (12, 34, 56)) + + +@pytest.mark.parametrize('response', ['error Can not load asset', 'Error: unavailable', None, '']) +def test_failed_spawn_stops_before_object_configuration(client, response): + """Never send physics or mobility commands for an actor that did not spawn.""" + client.client.request.return_value = response + with pytest.raises(RuntimeError, match='MissingBox'): + Communicator(client).spawn_object('MissingBox', '/Game/Missing.Missing_C', (0, 0, 0), (0, 0, 0)) + assert commands(client) == ['vset /objects/spawn_bp_asset /Game/Missing.Missing_C MissingBox'] + + +@pytest.mark.parametrize('response', ['ok', 'Box']) +def test_spawn_accepts_success_and_actor_name_responses(client, response): + """The official packaged backend responds with the spawned actor name.""" + client.client.request.return_value = response + client.spawn_bp_asset('/Game/Box.Box_C', 'Box') + + +@pytest.mark.parametrize('run_time', [True, False]) +def test_world_generation_preserves_palette_after_placement(client, tmp_path, run_time): + """Static city assets retain their configured semantic colors and final pose.""" + world = tmp_path / 'world.json' + assets = tmp_path / 'assets.json' + world.write_text(json.dumps({'nodes': [{ + 'id': 'Building', 'instance_name': 'BuildingModel', + 'properties': {'location': {'x': 10, 'y': 20, 'z': 0}, + 'orientation': {'pitch': 0, 'yaw': 90, 'roll': 0}, + 'scale': {'x': 2, 'y': 3, 'z': 4}}, + }]}), encoding='utf-8') + assets.write_text(json.dumps({ + 'BuildingModel': {'asset_path': '/Game/Building.Building_C', 'color': 'building'}, + 'colors': {'building': '(R=12,G=34,B=56)'}, + }), encoding='utf-8') + assert Communicator(client).generate_world(world, assets, run_time=run_time) == {'Building'} + sent = commands(client) + label = 'vset /object/Building/color 12 34 56' + if run_time: + assert sent[-1] == label + else: + assert label not in sent