docs(rpent): rename replay figures to Flash Mode

rpent/flash/README.md

@@ -0,0 +1,29 @@
+# Flash Mode figures
+
+Run `python plot.py` with matplotlib and numpy installed to regenerate both PNGs.
+The CSV files are the complete plotting inputs, so future label changes do not
+require access to private experiment logs.
+
+These figures preserve the published comparison; they are not a new evaluation.
+The full comparison has 581/800 Flash Mode successes, 500/800 Codex successes
+without reasoning, and 628/800 Codex successes with high reasoning. The Object
+comparison has 179/200 and 186/200 successes respectively.
+
+## Data sources and timing
+
+- `object.csv`: the original Object plotting CSV, with the method renamed.
+- `libero_pro.csv`: Flash success counts transcribed from the original figure's
+  integer bars and checked against the four family totals (149, 179, 139, 114).
+  Both Codex series come from the original 800-episode logs. For concatenated
+  transcript records, the last complete record is used; their success counts
+  reproduce the published figure.
+- Flash execution times come from the corresponding source episode's recorded
+  tool durations, checked against the original plotted bars. The original zero
+  bars for Spatial task 7 and Long task 9 are retained as published, rather than
+  replaced with timings from another run. Empty timing cells correspond to the
+  two unavailable plans: Goal swap 0 and Long swap 9.
+
+Codex timing is the mean planner duration across ten seeds per task. Flash timing
+is a single source-episode tool duration per plan where represented, not the mean
+of ten Flash evaluations. Model/service startup is excluded. Success rates use
+the complete evaluation matrix, counting the two missing plans as failures.

rpent/flash/libero_pro.csv

@@ -0,0 +1,81 @@
+family,task,flash_success,codex_success,codex_high_success,flash_execution_s,codex_execution_s,codex_high_execution_s
+spatial,task_t0,7,7,7,34.36000000000001,343.7,567.62
+spatial,task_t1,9,9,9,74.77,399.02,516.11
+spatial,task_t2,10,9,10,34.89000000000001,322.17,310.28
+spatial,task_t3,7,8,3,15.859999999999998,219.8,502.88
+spatial,task_t4,5,8,10,138.41,496.98,449.86
+spatial,task_t5,10,10,10,34.14,360.8,331.49
+spatial,task_t6,10,10,10,84.01,326.0,499.33
+spatial,task_t7,2,7,9,0.0,340.4,504.71999999999997
+spatial,task_t8,9,9,10,40.86,291.0,407.78
+spatial,task_t9,10,9,10,43.49,345.57,743.02
+spatial,swap_t0,7,7,10,35.65999999999999,289.97,449.84
+spatial,swap_t1,10,6,10,83.48,361.7,460.07
+spatial,swap_t2,9,8,10,26.450000000000003,271.34,346.01
+spatial,swap_t3,6,4,9,105.10000000000002,449.46999999999997,576.6
+spatial,swap_t4,2,8,9,119.37999999999998,480.42,484.31
+spatial,swap_t5,10,10,9,38.53,353.24,569.61
+spatial,swap_t6,1,3,4,69.16,598.69,948.45
+spatial,swap_t7,10,10,10,36.2,358.15,434.79
+spatial,swap_t8,10,6,10,48.519999999999996,392.43,604.77
+spatial,swap_t9,5,5,7,140.88,592.7,837.71
+object,task_t0,10,10,10,37.78,268.62,338.26
+object,task_t1,8,9,10,75.67,278.77,365.23
+object,task_t2,4,10,10,25.69,256.74,409.56
+object,task_t3,8,10,9,14.069999999999999,210.36,413.93
+object,task_t4,8,9,8,31.22,337.57,681.86
+object,task_t5,10,10,10,27.39,242.49,331.84
+object,task_t6,8,9,8,18.959999999999997,278.86,648.5
+object,task_t7,10,10,10,44.419999999999995,304.15,421.64
+object,task_t8,10,8,10,30.59,261.71999999999997,285.24
+object,task_t9,10,9,10,30.5,227.05,317.58
+object,swap_t0,9,10,9,47.95,294.31,427.5
+object,swap_t1,10,9,10,39.370000000000005,278.08,545.22
+object,swap_t2,9,10,10,41.92999999999999,237.71,386.2
+object,swap_t3,9,10,10,63.230000000000004,316.24,349.14
+object,swap_t4,10,7,10,50.14000000000001,300.79,300.83
+object,swap_t5,8,8,9,51.57,341.32,329.6
+object,swap_t6,10,10,10,51.05,287.54,355.24
+object,swap_t7,8,9,10,37.48,267.33,219.49
+object,swap_t8,10,10,10,42.43000000000001,315.65000000000003,363.93
+object,swap_t9,10,9,10,56.86,366.37,320.9
+goal,task_t0,5,1,9,107.75,290.62,281.13
+goal,task_t1,6,7,10,139.43,340.15,336.36
+goal,task_t2,6,5,6,14.930000000000003,344.45,486.67
+goal,task_t3,10,2,2,58.67000000000001,136.69,713.5699999999999
+goal,task_t4,5,0,9,42.56999999999999,38.55,391.21000000000004
+goal,task_t5,9,0,10,26.16,38.19,342.78
+goal,task_t6,6,0,4,28.1,38.29,776.96
+goal,task_t7,9,0,10,9.09,38.81,171.88
+goal,task_t8,8,0,7,49.43000000000001,38.38,502.26
+goal,task_t9,10,0,10,30.07,38.54,311.49
+goal,swap_t0,0,0,2,,312.06,763.34
+goal,swap_t1,2,5,6,135.67999999999998,518.64,686.47
+goal,swap_t2,6,7,4,39.22,276.93,597.39
+goal,swap_t3,9,6,9,78.07999999999998,360.66,407.03000000000003
+goal,swap_t4,10,10,10,42.699999999999996,271.56,358.52
+goal,swap_t5,4,0,3,53.65,827.54,1064.83
+goal,swap_t6,10,9,6,41.48,290.25,625.38
+goal,swap_t7,10,10,9,10.26,97.41,172.8
+goal,swap_t8,9,9,6,25.75,244.49,203.59
+goal,swap_t9,5,8,1,145.8,301.07,608.11
+10,task_t0,10,6,10,57.03,430.32,526.4300000000001
+10,task_t1,10,9,10,65.58,415.58,503.36
+10,task_t2,0,0,10,44.14999999999999,591.93,938.31
+10,task_t3,3,4,2,144.37,498.85,860.0
+10,task_t4,1,5,2,243.58999999999997,532.97,1454.8
+10,task_t5,8,2,10,7.949999999999999,349.52,347.14
+10,task_t6,9,8,5,109.75999999999999,539.75,672.3
+10,task_t7,9,4,9,102.73,494.89,563.42
+10,task_t8,10,5,10,9.41,184.8,380.85
+10,task_t9,0,0,0,0.0,539.17,1032.42
+10,swap_t0,9,9,9,152.15,477.2,587.8199999999999
+10,swap_t1,8,6,8,18.59,564.12,498.61
+10,swap_t2,2,4,6,17.89,404.08,664.9
+10,swap_t3,4,1,6,28.279999999999998,516.85,762.6
+10,swap_t4,2,3,4,56.48,617.49,1005.39
+10,swap_t5,10,4,7,7.809999999999999,367.84000000000003,402.29
+10,swap_t6,8,5,7,16.22,535.1800000000001,682.34
+10,swap_t7,10,7,10,87.10000000000001,587.32,493.84000000000003
+10,swap_t8,1,0,1,110.77,417.83,1035.04
+10,swap_t9,0,0,0,,359.17,767.69

rpent/flash/object.csv

@@ -0,0 +1,21 @@
+task,flash_success,codex_success,flash_execution_s,codex_execution_s
+swap_t0,9,10,47.95,294.31
+swap_t1,10,9,39.370000000000005,278.08
+swap_t2,9,10,41.92999999999999,237.71000000000004
+swap_t3,9,10,63.230000000000004,316.24
+swap_t4,10,7,50.14000000000001,300.79
+swap_t5,8,8,51.57,341.32
+swap_t6,10,10,51.05,287.54
+swap_t7,8,9,37.48,267.33000000000004
+swap_t8,10,10,42.43000000000001,315.65
+swap_t9,10,9,56.86,366.37
+task_t0,10,10,37.78,268.62
+task_t1,8,9,75.67,278.7699999999999
+task_t2,4,10,25.69,256.74
+task_t3,8,10,14.069999999999999,210.35999999999999
+task_t4,8,9,31.22,337.57
+task_t5,10,10,27.39,242.49
+task_t6,8,9,18.959999999999997,278.86
+task_t7,10,10,44.419999999999995,304.15
+task_t8,10,8,30.59,261.71999999999997
+task_t9,10,9,30.5,227.05

rpent/flash/plot.py

@@ -0,0 +1,105 @@
+"""Rebuild the published LIBERO comparisons using the Flash Mode name.
+
+Run with Python, matplotlib, and numpy installed. All plotting inputs are in
+the adjacent CSV files; no benchmark logs or network access are required.
+"""
+
+import csv
+from pathlib import Path
+
+import matplotlib
+
+matplotlib.use("Agg")
+import matplotlib.pyplot as plt
+import numpy as np
+
+ROOT = Path(__file__).resolve().parent
+COLORS = ("#277DA1", "#F8961E", "#7B2CBF")
+LABELS = ("Flash Mode", "Codex (no reasoning)", "Codex (reasoning high)")
+
+
+def read_rows(filename):
+    with (ROOT / filename).open() as handle:
+        return list(csv.DictReader(handle))
+
+
+def values(rows, key, scale=1):
+    return [float(row[key]) * scale if row[key] else np.nan for row in rows]
+
+
+def decorate(ax, labels):
+    ax.set_xticks(np.arange(len(labels)), labels, rotation=55, ha="right", fontsize=8)
+    ax.axvline(9.5, color="#777777", linewidth=1, alpha=0.6)
+    ax.grid(axis="y", alpha=0.2)
+
+
+def plot_full():
+    rows = read_rows("libero_pro.csv")
+    assert [sum(int(row[key]) for row in rows) for key in
+            ("flash_success", "codex_success", "codex_high_success")] == [581, 500, 628]
+    fig, axes = plt.subplots(4, 2, figsize=(18, 18), constrained_layout=True)
+    width = 0.25
+    for index, (family, label) in enumerate(
+        (("spatial", "Spatial"), ("object", "Object"), ("goal", "Goal"), ("10", "Long"))
+    ):
+        group = [row for row in rows if row["family"] == family]
+        x = np.arange(len(group))
+        for method, prefix in enumerate(("flash", "codex", "codex_high")):
+            for col, (metric, scale) in enumerate((("success", 10), ("execution_s", 1))):
+                axes[index, col].bar(
+                    x + (method - 1) * width, values(group, f"{prefix}_{metric}", scale),
+                    width, color=COLORS[method], label=LABELS[method],
+                )
+        success = sum(int(row["flash_success"]) for row in group)
+        axes[index, 0].set_title(f"{label}: success rate — Flash Mode {success}/200")
+        axes[index, 0].set_ylabel("Success rate (%)")
+        axes[index, 0].set_ylim(0, 112)
+        axes[index, 1].set_title(f"{label}: mean execution/planner time")
+        axes[index, 1].set_ylabel("Seconds per episode")
+        for i, row in enumerate(group):
+            if not row["flash_execution_s"]:
+                axes[index, 0].text(i - width, 4, "No plan (0/10)", rotation=90,
+                                    fontsize=7, ha="center", va="bottom")
+                axes[index, 1].text(i - width, 4, "N/A", rotation=90,
+                                    fontsize=7, ha="center", va="bottom")
+        for ax in axes[index]:
+            decorate(ax, [row["task"] for row in group])
+    for ax in axes[0]:
+        ax.legend(loc="upper right", fontsize=8, ncol=3)
+    fig.suptitle("Flash Mode vs. Codex — LIBERO-PRO (task/swap, 10 seeds per task)", fontsize=17)
+    fig.savefig(ROOT / "flash_libero_pro_performance_time.png", dpi=180)
+    plt.close(fig)
+
+
+def plot_object():
+    rows = read_rows("object.csv")
+    fig, axes = plt.subplots(2, 1, figsize=(15, 9), constrained_layout=True)
+    x = np.arange(len(rows))
+    width = 0.38
+    for method, prefix in enumerate(("flash", "codex")):
+        for col, metric in enumerate(("success", "execution_s")):
+            axes[col].bar(x + (method - 0.5) * width, values(rows, f"{prefix}_{metric}"),
+                          width, color=COLORS[method], label=LABELS[method])
+    axes[0].set_ylim(0, 11.3)
+    axes[0].set_ylabel("Successful episodes (out of 10)")
+    axes[0].set_title("Object tasks: per-task performance")
+    axes[0].text(0.01, 0.96, "Overall: Flash Mode 179/200 (89.5%) · Codex 186/200 (93.0%)",
+                 transform=axes[0].transAxes, va="top", fontsize=11)
+    axes[0].legend(loc="lower right")
+    axes[1].set_ylabel("Execution time per episode (seconds)")
+    axes[1].set_title("Object tasks: execution time (service startup excluded)")
+    flash_mean = np.mean(values(rows, "flash_execution_s"))
+    codex_mean = np.mean(values(rows, "codex_execution_s"))
+    axes[1].text(0.01, 0.96, f"Mean: Flash Mode {flash_mean:.1f}s · Codex {codex_mean:.1f}s",
+                 transform=axes[1].transAxes, va="top", fontsize=11)
+    axes[1].legend(loc="upper right")
+    for ax in axes:
+        decorate(ax, [row["task"] for row in rows])
+    fig.suptitle("Flash Mode vs. Codex (no reasoning) — LIBERO Object", fontsize=15)
+    fig.savefig(ROOT / "flash_object_performance_time.png", dpi=180)
+    plt.close(fig)
+
+
+if __name__ == "__main__":
+    plot_full()
+    plot_object()