diff --git a/rpent/flash/README.md b/rpent/flash/README.md new file mode 100644 index 0000000..f5a9dbb --- /dev/null +++ b/rpent/flash/README.md @@ -0,0 +1,29 @@ +# Flash Mode figures + +Run `python plot.py` with matplotlib and numpy installed to regenerate both PNGs. +The CSV files are the complete plotting inputs, so future label changes do not +require access to private experiment logs. + +These figures preserve the published comparison; they are not a new evaluation. +The full comparison has 581/800 Flash Mode successes, 500/800 Codex successes +without reasoning, and 628/800 Codex successes with high reasoning. The Object +comparison has 179/200 and 186/200 successes respectively. + +## Data sources and timing + +- `object.csv`: the original Object plotting CSV, with the method renamed. +- `libero_pro.csv`: Flash success counts transcribed from the original figure's + integer bars and checked against the four family totals (149, 179, 139, 114). + Both Codex series come from the original 800-episode logs. For concatenated + transcript records, the last complete record is used; their success counts + reproduce the published figure. +- Flash execution times come from the corresponding source episode's recorded + tool durations, checked against the original plotted bars. The original zero + bars for Spatial task 7 and Long task 9 are retained as published, rather than + replaced with timings from another run. Empty timing cells correspond to the + two unavailable plans: Goal swap 0 and Long swap 9. + +Codex timing is the mean planner duration across ten seeds per task. Flash timing +is a single source-episode tool duration per plan where represented, not the mean +of ten Flash evaluations. Model/service startup is excluded. Success rates use +the complete evaluation matrix, counting the two missing plans as failures. diff --git a/rpent/flash/flash_libero_pro_performance_time.png b/rpent/flash/flash_libero_pro_performance_time.png new file mode 100644 index 0000000..6e2bad9 Binary files /dev/null and b/rpent/flash/flash_libero_pro_performance_time.png differ diff --git a/rpent/flash/flash_object_performance_time.png b/rpent/flash/flash_object_performance_time.png new file mode 100644 index 0000000..a74b91b Binary files /dev/null and b/rpent/flash/flash_object_performance_time.png differ diff --git a/rpent/flash/libero_pro.csv b/rpent/flash/libero_pro.csv new file mode 100644 index 0000000..3dba606 --- /dev/null +++ b/rpent/flash/libero_pro.csv @@ -0,0 +1,81 @@ +family,task,flash_success,codex_success,codex_high_success,flash_execution_s,codex_execution_s,codex_high_execution_s +spatial,task_t0,7,7,7,34.36000000000001,343.7,567.62 +spatial,task_t1,9,9,9,74.77,399.02,516.11 +spatial,task_t2,10,9,10,34.89000000000001,322.17,310.28 +spatial,task_t3,7,8,3,15.859999999999998,219.8,502.88 +spatial,task_t4,5,8,10,138.41,496.98,449.86 +spatial,task_t5,10,10,10,34.14,360.8,331.49 +spatial,task_t6,10,10,10,84.01,326.0,499.33 +spatial,task_t7,2,7,9,0.0,340.4,504.71999999999997 +spatial,task_t8,9,9,10,40.86,291.0,407.78 +spatial,task_t9,10,9,10,43.49,345.57,743.02 +spatial,swap_t0,7,7,10,35.65999999999999,289.97,449.84 +spatial,swap_t1,10,6,10,83.48,361.7,460.07 +spatial,swap_t2,9,8,10,26.450000000000003,271.34,346.01 +spatial,swap_t3,6,4,9,105.10000000000002,449.46999999999997,576.6 +spatial,swap_t4,2,8,9,119.37999999999998,480.42,484.31 +spatial,swap_t5,10,10,9,38.53,353.24,569.61 +spatial,swap_t6,1,3,4,69.16,598.69,948.45 +spatial,swap_t7,10,10,10,36.2,358.15,434.79 +spatial,swap_t8,10,6,10,48.519999999999996,392.43,604.77 +spatial,swap_t9,5,5,7,140.88,592.7,837.71 +object,task_t0,10,10,10,37.78,268.62,338.26 +object,task_t1,8,9,10,75.67,278.77,365.23 +object,task_t2,4,10,10,25.69,256.74,409.56 +object,task_t3,8,10,9,14.069999999999999,210.36,413.93 +object,task_t4,8,9,8,31.22,337.57,681.86 +object,task_t5,10,10,10,27.39,242.49,331.84 +object,task_t6,8,9,8,18.959999999999997,278.86,648.5 +object,task_t7,10,10,10,44.419999999999995,304.15,421.64 +object,task_t8,10,8,10,30.59,261.71999999999997,285.24 +object,task_t9,10,9,10,30.5,227.05,317.58 +object,swap_t0,9,10,9,47.95,294.31,427.5 +object,swap_t1,10,9,10,39.370000000000005,278.08,545.22 +object,swap_t2,9,10,10,41.92999999999999,237.71,386.2 +object,swap_t3,9,10,10,63.230000000000004,316.24,349.14 +object,swap_t4,10,7,10,50.14000000000001,300.79,300.83 +object,swap_t5,8,8,9,51.57,341.32,329.6 +object,swap_t6,10,10,10,51.05,287.54,355.24 +object,swap_t7,8,9,10,37.48,267.33,219.49 +object,swap_t8,10,10,10,42.43000000000001,315.65000000000003,363.93 +object,swap_t9,10,9,10,56.86,366.37,320.9 +goal,task_t0,5,1,9,107.75,290.62,281.13 +goal,task_t1,6,7,10,139.43,340.15,336.36 +goal,task_t2,6,5,6,14.930000000000003,344.45,486.67 +goal,task_t3,10,2,2,58.67000000000001,136.69,713.5699999999999 +goal,task_t4,5,0,9,42.56999999999999,38.55,391.21000000000004 +goal,task_t5,9,0,10,26.16,38.19,342.78 +goal,task_t6,6,0,4,28.1,38.29,776.96 +goal,task_t7,9,0,10,9.09,38.81,171.88 +goal,task_t8,8,0,7,49.43000000000001,38.38,502.26 +goal,task_t9,10,0,10,30.07,38.54,311.49 +goal,swap_t0,0,0,2,,312.06,763.34 +goal,swap_t1,2,5,6,135.67999999999998,518.64,686.47 +goal,swap_t2,6,7,4,39.22,276.93,597.39 +goal,swap_t3,9,6,9,78.07999999999998,360.66,407.03000000000003 +goal,swap_t4,10,10,10,42.699999999999996,271.56,358.52 +goal,swap_t5,4,0,3,53.65,827.54,1064.83 +goal,swap_t6,10,9,6,41.48,290.25,625.38 +goal,swap_t7,10,10,9,10.26,97.41,172.8 +goal,swap_t8,9,9,6,25.75,244.49,203.59 +goal,swap_t9,5,8,1,145.8,301.07,608.11 +10,task_t0,10,6,10,57.03,430.32,526.4300000000001 +10,task_t1,10,9,10,65.58,415.58,503.36 +10,task_t2,0,0,10,44.14999999999999,591.93,938.31 +10,task_t3,3,4,2,144.37,498.85,860.0 +10,task_t4,1,5,2,243.58999999999997,532.97,1454.8 +10,task_t5,8,2,10,7.949999999999999,349.52,347.14 +10,task_t6,9,8,5,109.75999999999999,539.75,672.3 +10,task_t7,9,4,9,102.73,494.89,563.42 +10,task_t8,10,5,10,9.41,184.8,380.85 +10,task_t9,0,0,0,0.0,539.17,1032.42 +10,swap_t0,9,9,9,152.15,477.2,587.8199999999999 +10,swap_t1,8,6,8,18.59,564.12,498.61 +10,swap_t2,2,4,6,17.89,404.08,664.9 +10,swap_t3,4,1,6,28.279999999999998,516.85,762.6 +10,swap_t4,2,3,4,56.48,617.49,1005.39 +10,swap_t5,10,4,7,7.809999999999999,367.84000000000003,402.29 +10,swap_t6,8,5,7,16.22,535.1800000000001,682.34 +10,swap_t7,10,7,10,87.10000000000001,587.32,493.84000000000003 +10,swap_t8,1,0,1,110.77,417.83,1035.04 +10,swap_t9,0,0,0,,359.17,767.69 diff --git a/rpent/flash/object.csv b/rpent/flash/object.csv new file mode 100644 index 0000000..7ed9f51 --- /dev/null +++ b/rpent/flash/object.csv @@ -0,0 +1,21 @@ +task,flash_success,codex_success,flash_execution_s,codex_execution_s +swap_t0,9,10,47.95,294.31 +swap_t1,10,9,39.370000000000005,278.08 +swap_t2,9,10,41.92999999999999,237.71000000000004 +swap_t3,9,10,63.230000000000004,316.24 +swap_t4,10,7,50.14000000000001,300.79 +swap_t5,8,8,51.57,341.32 +swap_t6,10,10,51.05,287.54 +swap_t7,8,9,37.48,267.33000000000004 +swap_t8,10,10,42.43000000000001,315.65 +swap_t9,10,9,56.86,366.37 +task_t0,10,10,37.78,268.62 +task_t1,8,9,75.67,278.7699999999999 +task_t2,4,10,25.69,256.74 +task_t3,8,10,14.069999999999999,210.35999999999999 +task_t4,8,9,31.22,337.57 +task_t5,10,10,27.39,242.49 +task_t6,8,9,18.959999999999997,278.86 +task_t7,10,10,44.419999999999995,304.15 +task_t8,10,8,30.59,261.71999999999997 +task_t9,10,9,30.5,227.05 diff --git a/rpent/flash/plot.py b/rpent/flash/plot.py new file mode 100644 index 0000000..4c67e92 --- /dev/null +++ b/rpent/flash/plot.py @@ -0,0 +1,105 @@ +"""Rebuild the published LIBERO comparisons using the Flash Mode name. + +Run with Python, matplotlib, and numpy installed. All plotting inputs are in +the adjacent CSV files; no benchmark logs or network access are required. +""" + +import csv +from pathlib import Path + +import matplotlib + +matplotlib.use("Agg") +import matplotlib.pyplot as plt +import numpy as np + +ROOT = Path(__file__).resolve().parent +COLORS = ("#277DA1", "#F8961E", "#7B2CBF") +LABELS = ("Flash Mode", "Codex (no reasoning)", "Codex (reasoning high)") + + +def read_rows(filename): + with (ROOT / filename).open() as handle: + return list(csv.DictReader(handle)) + + +def values(rows, key, scale=1): + return [float(row[key]) * scale if row[key] else np.nan for row in rows] + + +def decorate(ax, labels): + ax.set_xticks(np.arange(len(labels)), labels, rotation=55, ha="right", fontsize=8) + ax.axvline(9.5, color="#777777", linewidth=1, alpha=0.6) + ax.grid(axis="y", alpha=0.2) + + +def plot_full(): + rows = read_rows("libero_pro.csv") + assert [sum(int(row[key]) for row in rows) for key in + ("flash_success", "codex_success", "codex_high_success")] == [581, 500, 628] + fig, axes = plt.subplots(4, 2, figsize=(18, 18), constrained_layout=True) + width = 0.25 + for index, (family, label) in enumerate( + (("spatial", "Spatial"), ("object", "Object"), ("goal", "Goal"), ("10", "Long")) + ): + group = [row for row in rows if row["family"] == family] + x = np.arange(len(group)) + for method, prefix in enumerate(("flash", "codex", "codex_high")): + for col, (metric, scale) in enumerate((("success", 10), ("execution_s", 1))): + axes[index, col].bar( + x + (method - 1) * width, values(group, f"{prefix}_{metric}", scale), + width, color=COLORS[method], label=LABELS[method], + ) + success = sum(int(row["flash_success"]) for row in group) + axes[index, 0].set_title(f"{label}: success rate — Flash Mode {success}/200") + axes[index, 0].set_ylabel("Success rate (%)") + axes[index, 0].set_ylim(0, 112) + axes[index, 1].set_title(f"{label}: mean execution/planner time") + axes[index, 1].set_ylabel("Seconds per episode") + for i, row in enumerate(group): + if not row["flash_execution_s"]: + axes[index, 0].text(i - width, 4, "No plan (0/10)", rotation=90, + fontsize=7, ha="center", va="bottom") + axes[index, 1].text(i - width, 4, "N/A", rotation=90, + fontsize=7, ha="center", va="bottom") + for ax in axes[index]: + decorate(ax, [row["task"] for row in group]) + for ax in axes[0]: + ax.legend(loc="upper right", fontsize=8, ncol=3) + fig.suptitle("Flash Mode vs. Codex — LIBERO-PRO (task/swap, 10 seeds per task)", fontsize=17) + fig.savefig(ROOT / "flash_libero_pro_performance_time.png", dpi=180) + plt.close(fig) + + +def plot_object(): + rows = read_rows("object.csv") + fig, axes = plt.subplots(2, 1, figsize=(15, 9), constrained_layout=True) + x = np.arange(len(rows)) + width = 0.38 + for method, prefix in enumerate(("flash", "codex")): + for col, metric in enumerate(("success", "execution_s")): + axes[col].bar(x + (method - 0.5) * width, values(rows, f"{prefix}_{metric}"), + width, color=COLORS[method], label=LABELS[method]) + axes[0].set_ylim(0, 11.3) + axes[0].set_ylabel("Successful episodes (out of 10)") + axes[0].set_title("Object tasks: per-task performance") + axes[0].text(0.01, 0.96, "Overall: Flash Mode 179/200 (89.5%) · Codex 186/200 (93.0%)", + transform=axes[0].transAxes, va="top", fontsize=11) + axes[0].legend(loc="lower right") + axes[1].set_ylabel("Execution time per episode (seconds)") + axes[1].set_title("Object tasks: execution time (service startup excluded)") + flash_mean = np.mean(values(rows, "flash_execution_s")) + codex_mean = np.mean(values(rows, "codex_execution_s")) + axes[1].text(0.01, 0.96, f"Mean: Flash Mode {flash_mean:.1f}s · Codex {codex_mean:.1f}s", + transform=axes[1].transAxes, va="top", fontsize=11) + axes[1].legend(loc="upper right") + for ax in axes: + decorate(ax, [row["task"] for row in rows]) + fig.suptitle("Flash Mode vs. Codex (no reasoning) — LIBERO Object", fontsize=15) + fig.savefig(ROOT / "flash_object_performance_time.png", dpi=180) + plt.close(fig) + + +if __name__ == "__main__": + plot_full() + plot_object() diff --git a/rpent/task_card/task_card_libero_pro_performance_time.png b/rpent/task_card/task_card_libero_pro_performance_time.png deleted file mode 100644 index 017c779..0000000 Binary files a/rpent/task_card/task_card_libero_pro_performance_time.png and /dev/null differ diff --git a/rpent/task_card/task_card_object_performance_time.png b/rpent/task_card/task_card_object_performance_time.png deleted file mode 100644 index 25cbd14..0000000 Binary files a/rpent/task_card/task_card_object_performance_time.png and /dev/null differ