Give each vehicle a camera, and a way to see the world without ArduPilot

CMakeLists.txt

@@ -33,6 +33,17 @@ FetchContent_Declare(httplib
 )
 FetchContent_MakeAvailable(httplib)
 
+# --- stb (single-header JPEG writer for the camera feed) --------------------------------------
+# Header-only and dependency-free, which is the point: the render path must not
+# drag an image library into a physics server that runs on Fargate.
+FetchContent_Declare(stb
+    GIT_REPOSITORY https://github.com/nothings/stb.git
+    GIT_TAG        f0569113c93ad095470c54bf34a17b36646bbbb5
+)
+FetchContent_MakeAvailable(stb)
+add_library(stb_image_write INTERFACE)
+target_include_directories(stb_image_write SYSTEM INTERFACE ${stb_SOURCE_DIR})
+
 # --- coverage instrumentation (our targets only; third-party stays clean) --------------------
 # Everything declared after this point picks up --coverage; Jolt/httplib are declared above.
 option(SKYSIM_COVERAGE "Instrument skysim targets for gcov/gcovr" OFF)
@@ -73,6 +84,15 @@ target_link_libraries(skysim_core PRIVATE Jolt)
 target_compile_options(skysim_core PRIVATE -Wall -Wextra -Wshadow)
 
 # --- terrain: OBJ -> MeshShape cooker + tile streamer ----------------------------------------
+add_library(skysim_render STATIC
+    src/render/renderer.cpp
+    src/render/raster.cpp
+    src/render/render_service.cpp
+    src/render/jpeg.cpp
+)
+target_include_directories(skysim_render PUBLIC src)
+target_link_libraries(skysim_render PUBLIC skysim_core PRIVATE stb_image_write)
+
 add_library(skysim_terrain STATIC
     src/terrain/cook.cpp
     src/terrain/tile_streamer.cpp
@@ -84,7 +104,7 @@ target_compile_options(skysim_terrain PRIVATE -Wall -Wextra -Wshadow)
 # --- skysim server --------------------------------------------------------------------------
 add_executable(skysim src/main.cpp)
 target_include_directories(skysim PRIVATE src)
-target_link_libraries(skysim PRIVATE skysim_protocol skysim_core skysim_vehicle skysim_api skysim_terrain)
+target_link_libraries(skysim PRIVATE skysim_protocol skysim_core skysim_vehicle skysim_api skysim_terrain skysim_render)
 target_compile_options(skysim PRIVATE -Wall -Wextra -Wshadow)
 
 # --- world-step benchmark (performance) ------------------------------------------------------
@@ -101,6 +121,10 @@ target_link_libraries(reply_bench PRIVATE skysim_protocol Threads::Threads)
 target_compile_options(reply_bench PRIVATE -Wall -Wextra -Wshadow)
 
 # --- tile cooker ------------------------------------------------------------------------------
+add_executable(render_probe tools/render_probe/main.cpp)
+target_include_directories(render_probe PRIVATE src)
+target_link_libraries(render_probe PRIVATE skysim_render skysim_core)
+
 add_executable(tile_cooker tools/cooker/main.cpp)
 target_include_directories(tile_cooker PRIVATE src)
 target_link_libraries(tile_cooker PRIVATE skysim_terrain)
@@ -139,6 +163,11 @@ target_link_libraries(test_collision PRIVATE skysim_core skysim_terrain)
 target_compile_options(test_collision PRIVATE -Wall -Wextra -Wshadow)
 add_test(NAME collision COMMAND test_collision)
 
+add_executable(test_render tests/test_render.cpp)
+target_link_libraries(test_render PRIVATE skysim_render skysim_core skysim_terrain)
+target_compile_options(test_render PRIVATE -Wall -Wextra -Wshadow)
+add_test(NAME render COMMAND test_render)
+
 add_executable(test_endpoint tests/test_endpoint.cpp)
 target_link_libraries(test_endpoint PRIVATE skysim_protocol)
 target_compile_options(test_endpoint PRIVATE -Wall -Wextra -Wshadow)

README.md

@@ -84,6 +84,31 @@ Read in depth: [`docs/PROTOCOL.md`](docs/PROTOCOL.md) → [`docs/DESIGN.md`](doc
 
 ---
 
+## Getting started
+
+One command. It builds skysim, cooks a 1.2 km demo city, and renders it from four camera
+poses — the same rasteriser that feeds a vehicle's camera stream in flight, so what you
+get is what a drone would see:
+
+```bash
+tools/getting_started.sh
+```
+
+![Four views of the demo city rendered by skysim](.github/assets/skysim-getting-started.jpg)
+
+Images land in `build/getting_started/`. Needs the build tools, Python 3, and — on a
+clean checkout — network access, because CMake fetches JoltPhysics, cpp-httplib and stb.
+It does **not** need ArduPilot or a running simulator, which is the point: you can see
+the thing work before committing to the half hour the autopilot build takes.
+
+Once you do have ArduPilot (see below), the same script will fly a vehicle through that
+city and serve its camera live:
+
+```bash
+ARDUPILOT_ROOT=~/ardupilot tools/getting_started.sh --fly
+# then open http://127.0.0.1:8642/instances/0/camera.mjpg
+```
+
 ## Quick start
 
 **Prerequisites** — Ubuntu 22.04/24.04, CMake ≥ 3.24, Ninja, GCC 12+ or Clang 16+, and an
@@ -230,9 +255,12 @@ src/core/       frames.h (NED/FRD <-> Jolt), world.cpp (the ONLY Jolt-aware TU),
 src/vehicle/    motor lag + X-quad mixer + instance allocator / process manager
 src/terrain/    OBJ -> MeshShape cooker + proximity tile streamer
 src/api/        REST control plane (cpp-httplib)
+src/render/     threaded software rasteriser + JPEG/MJPEG camera feed per vehicle
 tools/cooker/   pretile.py (demo city) + osm_buildings.py (real city) + tile_cooker CLI
 tools/bench/    gated world, protocol, and UDP reply-path performance benchmarks
 tools/harness/  conformance / determinism / straggler / churn / collision / corridor
+tools/render_probe/  render one frame from a given camera pose, to a JPEG
+tools/getting_started.sh  build + cook + render the demo city (see "Getting started")
 tests/          unit tests + app_smoke.py driving the real binary
 ```
 

src/api/control_server.cpp

@@ -17,6 +17,10 @@ namespace skysim::api {
 
 namespace {
 
+// Two long-lived camera streams per vehicle, plus headroom for the control
+// plane, which must stay answerable while they run.
+constexpr size_t kWorkerThreads = 64;
+
 // {"launch_process":true} — absent key means false.
 bool parse_launch_process(const std::string &body) {
     const char *p = std::strstr(body.c_str(), "\"launch_process\"");
@@ -157,6 +161,16 @@ ControlServer::ControlServer(const std::string &bind_addr, int port, CommandQueu
     : impl_(std::make_unique<Impl>()) {
     auto &s = impl_->server;
 
+    // Camera streams hold a worker for their whole life.
+    //
+    // Each MJPEG response sits in its chunked provider until the client goes
+    // away, so it occupies one thread the entire time — and every simulated
+    // vehicle opens two (the core container's encoder and the recorder). At
+    // httplib's default pool of max(8, ncpu-1), four vehicles on a 2-vCPU task
+    // would consume every worker and /vehicles, /metrics and spawn would simply
+    // stop answering. Sized for streams rather than for cores.
+    s.new_task_queue = [] { return new httplib::ThreadPool(kWorkerThreads); };
+
     s.Post("/vehicles", [&queue](const httplib::Request &req, httplib::Response &res) {
         SpawnCommand cmd;
         cmd.request.launch_process = parse_launch_process(req.body);
@@ -265,6 +279,124 @@ ControlServer::ControlServer(const std::string &bind_addr, int port, CommandQueu
         res.set_content(out, "application/json");
     });
 
+    // The same two feeds addressed by ArduPilot instance.
+    //
+    // skysim's own vehicle ids are allocated internally and change when it
+    // restarts; the instance is what the gateway assigns and everything else
+    // already agrees on. A consumer that can only bake a URL into a config —
+    // an ffmpeg command line, say — needs one that stays true.
+    auto resolve_instance = [snapshots](const std::string &instance) -> uint32_t {
+        if (!snapshots.vehicles) {
+            return 0;
+        }
+        for (const auto &v : snapshots.vehicles()) {
+            if (std::to_string(v.instance) == instance) {
+                return v.id;
+            }
+        }
+        return 0;
+    };
+
+    s.Get(R"(/instances/(\d+)/camera.jpg)", [snapshots, resolve_instance](const httplib::Request &req,
+                                                                           httplib::Response &res) {
+        const uint32_t id = resolve_instance(req.matches[1].str());
+        if (id == 0 || !snapshots.camera_frame) {
+            res.status = 404;
+            res.set_content("{\"error\":\"no such instance\"}", "application/json");
+            return;
+        }
+        std::vector<uint8_t> jpeg = snapshots.camera_frame(id);
+        if (jpeg.empty()) {
+            res.status = 404;
+            res.set_content("{\"error\":\"no frame\"}", "application/json");
+            return;
+        }
+        res.set_content(reinterpret_cast<const char *>(jpeg.data()), jpeg.size(), "image/jpeg");
+    });
+
+    s.Get(R"(/instances/(\d+)/camera.mjpg)", [snapshots, resolve_instance](const httplib::Request &req,
+                                                                            httplib::Response &res) {
+        const std::string instance = req.matches[1].str();
+        if (!snapshots.camera_frame) {
+            res.status = 404;
+            res.set_content("{\"error\":\"camera disabled\"}", "application/json");
+            return;
+        }
+        res.set_chunked_content_provider(
+            "multipart/x-mixed-replace; boundary=skysimframe",
+            [snapshots, resolve_instance, instance](size_t, httplib::DataSink &sink) {
+                // Re-resolved per frame, so a skysim restart mid-recording picks
+                // the vehicle back up instead of streaming nothing forever.
+                const uint32_t id = resolve_instance(instance);
+                std::vector<uint8_t> jpeg = id != 0 ? snapshots.camera_frame(id) : std::vector<uint8_t>{};
+                if (!jpeg.empty()) {
+                    char head[128];
+                    const int n = std::snprintf(head, sizeof(head),
+                                                "--skysimframe\r\nContent-Type: image/jpeg\r\n"
+                                                "Content-Length: %zu\r\n\r\n",
+                                                jpeg.size());
+                    sink.write(head, static_cast<size_t>(n));
+                    sink.write(reinterpret_cast<const char *>(jpeg.data()), jpeg.size());
+                    sink.write("\r\n", 2);
+                }
+                std::this_thread::sleep_for(std::chrono::milliseconds(40));
+                return true;
+            });
+    });
+
+    // One frame, for a poll-based viewer or a quick look with curl.
+    s.Get(R"(/vehicles/(\d+)/camera.jpg)", [snapshots](const httplib::Request &req, httplib::Response &res) {
+        if (!snapshots.camera_frame) {
+            res.status = 404;
+            res.set_content("{\"error\":\"camera disabled\"}", "application/json");
+            return;
+        }
+        // strtoul, not stoul: the route regex accepts any number of digits, and
+        // stoul throws on a value too big for the type. Saturating is the same
+        // answer as "no such vehicle" without the exception.
+        std::vector<uint8_t> jpeg = snapshots.camera_frame(
+            static_cast<uint32_t>(std::strtoul(req.matches[1].str().c_str(), nullptr, 10)));
+        if (jpeg.empty()) {
+            res.status = 404;
+            res.set_content("{\"error\":\"no frame\"}", "application/json");
+            return;
+        }
+        res.set_content(reinterpret_cast<const char *>(jpeg.data()), jpeg.size(), "image/jpeg");
+    });
+
+    // MJPEG, which is what the video pipeline consumes.
+    //
+    // A chunked multipart stream rather than a socket of raw frames because it
+    // crosses a container boundary and GStreamer, ffmpeg and a plain browser tab
+    // can all open it without agreeing on anything first.
+    s.Get(R"(/vehicles/(\d+)/camera.mjpg)", [snapshots](const httplib::Request &req, httplib::Response &res) {
+        if (!snapshots.camera_frame) {
+            res.status = 404;
+            res.set_content("{\"error\":\"camera disabled\"}", "application/json");
+            return;
+        }
+        const auto id = static_cast<uint32_t>(std::strtoul(req.matches[1].str().c_str(), nullptr, 10));
+        res.set_chunked_content_provider(
+            "multipart/x-mixed-replace; boundary=skysimframe",
+            [snapshots, id](size_t /*offset*/, httplib::DataSink &sink) {
+                std::vector<uint8_t> jpeg = snapshots.camera_frame(id);
+                if (!jpeg.empty()) {
+                    char head[128];
+                    const int n = std::snprintf(head, sizeof(head),
+                                                "--skysimframe\r\nContent-Type: image/jpeg\r\n"
+                                                "Content-Length: %zu\r\n\r\n",
+                                                jpeg.size());
+                    sink.write(head, static_cast<size_t>(n));
+                    sink.write(reinterpret_cast<const char *>(jpeg.data()), jpeg.size());
+                    sink.write("\r\n", 2);
+                }
+                // Poll a little faster than frames are produced, so the stream
+                // tracks the render rate instead of setting its own.
+                std::this_thread::sleep_for(std::chrono::milliseconds(40));
+                return true;
+            });
+    });
+
     s.Get("/metrics", [snapshots](const httplib::Request &, httplib::Response &res) {
         const MetricsInfo m = snapshots.metrics();
         char buf[512];

src/api/control_server.h

@@ -103,6 +103,10 @@ class ControlServer {
     struct Snapshots {
         std::function<std::vector<VehicleInfo>()> vehicles;
         std::function<MetricsInfo()> metrics;
+        // Latest camera frame for a vehicle as encoded JPEG; empty when the
+        // camera is off or that vehicle has not been rendered yet. Published by
+        // the tick thread like everything else here.
+        std::function<std::vector<uint8_t>(uint32_t)> camera_frame;
     };
 
     // Binds bind_addr:port and serves on its own thread. Throws on bind failure (startup

src/core/world.cpp

@@ -162,6 +162,7 @@ struct World::Impl {
     uint32_t next_id = 1;
     std::unordered_map<uint32_t, VehicleEntry> vehicles;
     std::vector<JPH::BodyID> static_bodies;
+    JPH::BodyID ground_body; // also in static_bodies; kept apart so drawing can name it
     std::unordered_map<uint32_t, JPH::BodyID> static_tiles; // streamed (M5)
     // Body indices of resident tiles, so contact attribution can distinguish a
     // building strike from an ordinary ground touch without a linear scan.
@@ -233,6 +234,7 @@ void World::add_ground_plane() {
     s.mRestitution = 0.0f;
     const JPH::BodyID id = impl_->bodies().CreateAndAddBody(s, JPH::EActivation::DontActivate);
     impl_->static_bodies.push_back(id);
+    impl_->ground_body = id;
 }
 
 namespace {
@@ -303,6 +305,47 @@ void World::remove_static_tile(uint32_t id) {
     impl_->static_tiles.erase(it);
 }
 
+std::vector<World::Triangle> World::collect_static_triangles() const {
+    std::vector<Triangle> out;
+    const JPH::BodyInterface &bodies = impl_->bodies();
+
+    // Both routes into the world: bulk-loaded tiles (and the ground slab) live in
+    // static_bodies, streamed ones in static_tiles. Missing either draws a world
+    // with holes in it exactly where the geometry came from the other path.
+    std::vector<JPH::BodyID> all;
+    all.reserve(impl_->static_bodies.size() + impl_->static_tiles.size());
+    all.insert(all.end(), impl_->static_bodies.begin(), impl_->static_bodies.end());
+    for (const auto &[id, body] : impl_->static_tiles) {
+        all.push_back(body);
+    }
+
+    for (const JPH::BodyID &body : all) {
+        const bool is_ground = body == impl_->ground_body;
+        const JPH::TransformedShape shape = bodies.GetTransformedShape(body);
+
+        // Jolt walks a shape's triangles in batches through this cursor; it is
+        // the same path the debug renderer uses.
+        JPH::Shape::GetTrianglesContext ctx;
+        constexpr int kBatch = 256;
+        JPH::Float3 verts[kBatch * 3];
+        shape.GetTrianglesStart(ctx, JPH::AABox::sBiggest(), JPH::RVec3::sZero());
+        for (;;) {
+            const int count = shape.GetTrianglesNext(ctx, kBatch, verts);
+            if (count == 0) {
+                break;
+            }
+            for (int i = 0; i < count; ++i) {
+                const JPH::Float3 &a = verts[i * 3 + 0];
+                const JPH::Float3 &b = verts[i * 3 + 1];
+                const JPH::Float3 &c = verts[i * 3 + 2];
+                out.push_back({from_jolt(JPH::Vec3(a.x, a.y, a.z)), from_jolt(JPH::Vec3(b.x, b.y, b.z)),
+                               from_jolt(JPH::Vec3(c.x, c.y, c.z)), is_ground});
+            }
+        }
+    }
+    return out;
+}
+
 void World::optimize_broadphase() { impl_->physics->OptimizeBroadPhase(); }
 
 double World::raycast(const Vec3 &origin_ned, const Vec3 &dir_ned, double max_dist_m, uint32_t ignore_vehicle_id,
@@ -331,6 +374,39 @@ double World::raycast(const Vec3 &origin_ned, const Vec3 &dir_ned, double max_di
     return -1.0;
 }
 
+World::RayHit World::raycast_surface(const Vec3 &origin_ned, const Vec3 &dir_ned, double max_dist_m,
+                                     uint32_t ignore_vehicle_id) const {
+    RayHit out;
+    const JPH::RVec3 origin(to_jolt(origin_ned));
+    const JPH::Vec3 dir = to_jolt(dir_ned) * static_cast<float>(max_dist_m);
+    JPH::RRayCast ray{origin, dir};
+    JPH::RayCastResult hit;
+    JPH::BodyID ignore;
+    if (ignore_vehicle_id != 0) {
+        auto it = impl_->vehicles.find(ignore_vehicle_id);
+        if (it != impl_->vehicles.end()) {
+            ignore = it->second.body;
+        }
+    }
+    const JPH::IgnoreSingleBodyFilter body_filter(ignore);
+    const JPH::BroadPhaseLayerFilter any_bp;
+    const JPH::ObjectLayerFilter any_object;
+    if (!impl_->physics->GetNarrowPhaseQuery().CastRay(ray, hit, any_bp, any_object, body_filter)) {
+        return out;
+    }
+
+    out.hit = true;
+    out.distance = hit.mFraction * max_dist_m;
+
+    // Ask the body that was hit which way its surface faces at the hit point.
+    const JPH::RVec3 point = ray.GetPointOnRay(hit.mFraction);
+    const JPH::Vec3 normal =
+        impl_->bodies().GetTransformedShape(hit.mBodyID).GetWorldSpaceSurfaceNormal(hit.mSubShapeID2, point);
+    out.normal_ned = from_jolt(normal);
+    out.is_ground = hit.mBodyID == impl_->ground_body;
+    return out;
+}
+
 std::vector<World::PathHit> World::sweep_path(const std::vector<Vec3> &waypoints_ned, double clearance_m) const {
     std::vector<PathHit> hits;
     if (waypoints_ned.size() < 2) {