Bin the RGB diodes beside the camera in /binned

src/dls_tiled/visr.py

@@ -614,9 +614,11 @@ async def binned(  # type: ignore
 ):
     """Fetch a folded representation of an array dataset.
 
-    Returns ``RedTotal``/``GreenTotal``/``BlueTotal`` as matrices with one row per
-    y bin and one column per x bin (the layout image plots expect), ``null`` for
-    bins no point fell into, ``x_limits``/``y_limits`` as the bin edges (one more
+    Returns ``RedTotal``/``GreenTotal``/``BlueTotal`` (the camera) and
+    ``DiodesRed``/``DiodesGreen``/``DiodesBlue`` (the RGB diodes, ``null`` if the
+    run has none) as matrices with one row per y bin and one column per x bin
+    (the layout image plots expect), ``null`` for bins no point fell into, the
+    diodes on the camera's bins, ``x_limits``/``y_limits`` as the bin edges (one more
     than the columns/rows), ``n_points``, the number of scan points used, and
     ``n_total``, the number the scan will have (``null`` if the run has no spec),
     and ``position_offset``: ``{"x": dx, "y": dy}`` when the recorded positions
@@ -699,6 +701,38 @@ def _conditional(request: Request, payload: bytes, media: str, tag: str) -> Resp
     return Response(payload, media_type=media, headers=headers)
 
 
+CAMERA_CHANNELS = ("RedTotal", "GreenTotal", "BlueTotal")
+
+# The RGB diodes, by their name in the /binned answer, with where runs record
+# them: the first path that exists wins. Fly scans write the PandA's per-frame
+# means into primary, step scans read the diodes into the event table.
+DIODE_CHANNELS = {
+    name: (
+        (f"spectroscopy_diodes-{colour}",),
+        ("internal", f"spectroscopy_diodes-{colour}"),
+        (colour,),
+    )
+    for name, colour in (
+        ("DiodesRed", "red"),
+        ("DiodesGreen", "green"),
+        ("DiodesBlue", "blue"),
+    )
+}
+
+
+async def read_optional_channel(root, uid, places) -> numpy.ndarray | None:
+    """A channel's values from the first of ``places`` under the run's primary
+    stream that exists, or None if the run records it in none of them."""
+    for place in places:
+        try:
+            found = await get_data(root, [uid, "primary", *place])
+        except NoEntry:
+            continue
+        if isinstance(found, (H5Dataset, numpy.ndarray)):
+            return numpy.asarray(found)
+    return None
+
+
 class ScanPoints(NamedTuple):
     """The points of a run ready to draw: each channel's values, their positions,
     how many there are, whether they were placed on the scan's grid, and the
@@ -714,10 +748,23 @@ class ScanPoints(NamedTuple):
 
 
 async def load_points(
-    root, uid, x_dim_index, y_dim_index, setpoints, slice_dim, grid=None
+    root,
+    uid,
+    x_dim_index,
+    y_dim_index,
+    setpoints,
+    slice_dim,
+    grid=None,
+    optional_channels=None,
 ):
     """Read a run's channels and positions, reconciled to one length and sliced.
 
+    ``optional_channels`` maps more channel names to where runs may record them
+    (see ``read_optional_channel``). One the run doesn't record is left out of
+    ``data``. They take no part in reconciling lengths, so a lagging one never
+    holds the camera channels back: it keeps only the points it has, and may be
+    shorter than the positions.
+
     Positions are the recorded readbacks, or the spec's setpoints with
     ``setpoints``. With a ``grid`` (for x and y as dimensions 0 and 1), the
     positions are placed on it (see ``place_on_grid``) before ``slice_dim`` is
@@ -844,6 +891,10 @@ def current_lengths() -> dict[str, int]:
         "GreenTotal": green_total,
         "BlueTotal": blue_total,
     }
+    for name, places in (optional_channels or {}).items():
+        found = await read_optional_channel(root, uid, places)
+        if found is not None:
+            data[name] = found[:common_len]
 
     readbacks = numpy.asarray(readbacks, dtype=float)
     on_grid, offset = False, None
@@ -887,7 +938,7 @@ def current_lengths() -> dict[str, int]:
             mask &= numpy.abs(readbacks[dim, :] - center) <= thickness
 
         readbacks = readbacks[:, mask]
-        data = {channel: d[mask] for channel, d in data.items()}
+        data = {channel: d[mask[: len(d)]] for channel, d in data.items()}
 
     # A point with no position on x or y (NaN: a run that only moved x has no
     # y readings, say) can't be placed in any bin, and numpy's default binning
@@ -971,6 +1022,7 @@ async def _binned_output(
         setpoints,
         slice_dim,
         grid if use_grid else None,
+        optional_channels=DIODE_CHANNELS,
     )
     data, x_positions, y_positions = points.data, points.x, points.y
     n_points, position_offset = points.n_points, points.offset
@@ -981,15 +1033,32 @@ async def _binned_output(
     def bin_channels() -> dict:
         """The CPU-bound part, run off the event loop."""
         binned_output: dict = {}
-        for channel in ("RedTotal", "GreenTotal", "BlueTotal"):
-            binned_channel = compute_binned_image(
+        camera = {
+            channel: compute_binned_image(
                 data[channel], x_positions, y_positions, **histogram2d_kwargs
             )
+            for channel in CAMERA_CHANNELS
+        }
+        for channel, binned_channel in camera.items():
             binned_output[channel] = image_rows_are_y(
                 binned_channel["img"], binned_channel["counts"]
             )
-        binned_output["x_limits"] = binned_channel["x"].tolist()
-        binned_output["y_limits"] = binned_channel["y"].tolist()
+        x_edges, y_edges = camera["RedTotal"]["x"], camera["RedTotal"]["y"]
+        binned_output["x_limits"] = x_edges.tolist()
+        binned_output["y_limits"] = y_edges.tolist()
+        # on the camera's bins, which the diodes may have fewer points than
+        for channel in DIODE_CHANNELS:
+            values = data.get(channel)
+            if values is None:
+                binned_output[channel] = None
+                continue
+            n = len(values)
+            binned_diode = compute_binned_image(
+                values, x_positions[:n], y_positions[:n], bins=[x_edges, y_edges]
+            )
+            binned_output[channel] = image_rows_are_y(
+                binned_diode["img"], binned_diode["counts"]
+            )
         return binned_output
 
     binned_output = await anyio.to_thread.run_sync(bin_channels)

tests/test_visr.py

@@ -1180,3 +1180,98 @@ async def lookup(segments):
     assert response.status_code == status, response.text
     if status == 503:
         assert response.headers["retry-after"] == "1"
+
+
+# the RGB diodes
+
+
+SQUARE_X, SQUARE_Y = [0.0, 1.0, 0.0, 1.0], [0.0, 0.0, 1.0, 1.0]
+DIODES = {"red": [10.0, 11.0, 12.0, 13.0], "green": [20.0] * 4, "blue": [30.0] * 4}
+
+
+def test_binned_has_null_diodes_for_a_run_without_them(client):
+    write_points(client, "camera-only", SQUARE_X, SQUARE_Y)
+
+    body = client.context.http_client.get(
+        "/api/v1/binned/camera-only", params={"width": 2, "height": 2}
+    ).json()
+
+    assert body["RedTotal"] == [[0.0, 1.0], [2.0, 3.0]]
+    assert body["DiodesRed"] is None
+    assert body["DiodesGreen"] is None
+    assert body["DiodesBlue"] is None
+
+
+@pytest.mark.parametrize("names", ["spectroscopy_diodes-{}", "{}"])
+def test_binned_bins_a_fly_scans_diodes_beside_the_camera(client, names):
+    write_points(client, "fly", SQUARE_X, SQUARE_Y)
+    primary = client["fly"]["primary"]
+    for colour, values in DIODES.items():
+        primary.write_array(numpy.array(values), key=names.format(colour))
+
+    body = client.context.http_client.get(
+        "/api/v1/binned/fly", params={"width": 2, "height": 2}
+    ).json()
+
+    assert body["DiodesRed"] == [[10.0, 11.0], [12.0, 13.0]]
+    assert body["DiodesGreen"] == [[20.0, 20.0], [20.0, 20.0]]
+    assert body["DiodesBlue"] == [[30.0, 30.0], [30.0, 30.0]]
+    assert body["RedTotal"] == [[0.0, 1.0], [2.0, 3.0]]
+
+
+def test_binned_bins_a_step_scans_diodes_from_its_event_table(client):
+    # a step scan reads positions and diodes into one table, primary/internal
+    import pandas
+
+    primary = client.create_container("step").create_container("primary")
+    primary.write_dataframe(
+        pandas.DataFrame(
+            {
+                "sample_stage-x": SQUARE_X,
+                "sample_stage-y": SQUARE_Y,
+                **{f"spectroscopy_diodes-{c}": v for c, v in DIODES.items()},
+            }
+        ),
+        key="internal",
+    )
+    for channel in ("RedTotal", "GreenTotal", "BlueTotal"):
+        primary.write_array(numpy.arange(4.0), key=channel)
+
+    response = client.context.http_client.get(
+        "/api/v1/binned/step", params={"width": 2, "height": 2}
+    )
+
+    assert response.status_code == 200, response.text
+    body = response.json()
+    assert body["DiodesRed"] == [[10.0, 11.0], [12.0, 13.0]]
+    assert body["RedTotal"] == [[0.0, 1.0], [2.0, 3.0]]
+
+
+def test_binned_diodes_behind_the_camera_hold_nothing_back(client):
+    write_points(client, "lagging", SQUARE_X, SQUARE_Y)
+    primary = client["lagging"]["primary"]
+    for colour, values in DIODES.items():
+        primary.write_array(
+            numpy.array(values[:3]), key=f"spectroscopy_diodes-{colour}"
+        )
+
+    body = client.context.http_client.get(
+        "/api/v1/binned/lagging", params={"width": 2, "height": 2}
+    ).json()
+
+    assert body["n_points"] == 4
+    assert body["RedTotal"] == [[0.0, 1.0], [2.0, 3.0]]
+    # on the camera's bins, the cell the diodes haven't reached empty
+    assert body["DiodesRed"] == [[10.0, 11.0], [12.0, None]]
+
+
+def test_binned_diodes_follow_the_scans_grid(client):
+    write_grid_scan(client, "grid", 9)
+    primary = client["grid"]["primary"]
+    for colour in DIODES:
+        primary.write_array(numpy.arange(9.0), key=f"spectroscopy_diodes-{colour}")
+
+    body = client.context.http_client.get("/api/v1/binned/grid").json()
+
+    assert numpy.shape(body["DiodesRed"]) == numpy.shape(body["RedTotal"])
+    assert body["DiodesRed"] == body["RedTotal"]