From e6ea7d4f19b986337259fc43aa9e385458abfa74 Mon Sep 17 00:00:00 2001 From: elkoled Date: Sat, 14 Nov 2026 13:44:22 -0800 Subject: [PATCH 01/21] safety replay on the rack --- alfred.toml | 61 ++++++- opendbc_repo | 2 +- .../selfdrive/pandad/tests/safety_replay.py | 166 ++++++++++++++++++ 3 files changed, 227 insertions(+), 2 deletions(-) create mode 100755 openpilot/selfdrive/pandad/tests/safety_replay.py diff --git a/alfred.toml b/alfred.toml index 9ba6ebeda2cd9f..6afdb227b9df1b 100644 --- a/alfred.toml +++ b/alfred.toml @@ -1,5 +1,5 @@ # a UP's DUT starts each job with an empty /data: these come back from master's last good job, as it left them -cache = ["/data/scons_cache"] +cache = ["/data/scons_cache", "/data/tmp/comma_download_cache"] [env] PYTHONPATH = "/data/ci/tree/openpilot" # the tree @@ -23,6 +23,14 @@ for p in msgq_repo/msgq opendbc_repo/opendbc rednose_repo/rednose teleoprtc_repo tools/op.sh build """ +[steps.flash-panda] +name = "Flash panda" +run = "python3 -c 'from openpilot.selfdrive.pandad.pandad import flash_panda; flash_panda()'" + +[steps.safety-replay] +name = "Safety replay" +run = "./openpilot/selfdrive/pandad/tests/safety_replay.py" + [steps.flash-chestnut] name = "Flash Chestnut" run = "sudo python3 openpilot/system/hardware/chestnut/flash.py" @@ -117,6 +125,57 @@ uses = "build" name = "Sensord" run = "./openpilot/system/sensord/tests/test_sensord.py" +# a tenth of the safety replay routes each, on the up-can stations and any free comma four +[jobs.safety_replay_0] +runs-on = "up-can" +env = { NUM_JOBS = "10", JOB_ID = "0" } +steps = [{ uses = "build" }, { uses = "flash-panda" }, { uses = "safety-replay" }] + +[jobs.safety_replay_1] +runs-on = "up-can" +env = { NUM_JOBS = "10", JOB_ID = "1" } +steps = [{ uses = "build" }, { uses = "flash-panda" }, { uses = "safety-replay" }] + +[jobs.safety_replay_2] +runs-on = "up-can" +env = { NUM_JOBS = "10", JOB_ID = "2" } +steps = [{ uses = "build" }, { uses = "flash-panda" }, { uses = "safety-replay" }] + +[jobs.safety_replay_3] +runs-on = "up-can" +env = { NUM_JOBS = "10", JOB_ID = "3" } +steps = [{ uses = "build" }, { uses = "flash-panda" }, { uses = "safety-replay" }] + +[jobs.safety_replay_4] +runs-on = "up-can" +env = { NUM_JOBS = "10", JOB_ID = "4" } +steps = [{ uses = "build" }, { uses = "flash-panda" }, { uses = "safety-replay" }] + +[jobs.safety_replay_5] +runs-on = "up-can" +env = { NUM_JOBS = "10", JOB_ID = "5" } +steps = [{ uses = "build" }, { uses = "flash-panda" }, { uses = "safety-replay" }] + +[jobs.safety_replay_6] +runs-on = "up-can" +env = { NUM_JOBS = "10", JOB_ID = "6" } +steps = [{ uses = "build" }, { uses = "flash-panda" }, { uses = "safety-replay" }] + +[jobs.safety_replay_7] +runs-on = "up-can" +env = { NUM_JOBS = "10", JOB_ID = "7" } +steps = [{ uses = "build" }, { uses = "flash-panda" }, { uses = "safety-replay" }] + +[jobs.safety_replay_8] +runs-on = "up-can" +env = { NUM_JOBS = "10", JOB_ID = "8" } +steps = [{ uses = "build" }, { uses = "flash-panda" }, { uses = "safety-replay" }] + +[jobs.safety_replay_9] +runs-on = "up-can" +env = { NUM_JOBS = "10", JOB_ID = "9" } +steps = [{ uses = "build" }, { uses = "flash-panda" }, { uses = "safety-replay" }] + # a comma four with a Chestnut on its aux port, which runs the big model [jobs.chestnut] runs-on = "chestnut" diff --git a/opendbc_repo b/opendbc_repo index 27fa1c5d8eed32..4b0d18fd0f8916 160000 --- a/opendbc_repo +++ b/opendbc_repo @@ -1 +1 @@ -Subproject commit 27fa1c5d8eed3238a0a78c52ff5abb6cdc807bd2 +Subproject commit 4b0d18fd0f8916ba61101f960e97515a7ae4622f diff --git a/openpilot/selfdrive/pandad/tests/safety_replay.py b/openpilot/selfdrive/pandad/tests/safety_replay.py new file mode 100755 index 00000000000000..6b1c713bd7ed68 --- /dev/null +++ b/openpilot/selfdrive/pandad/tests/safety_replay.py @@ -0,0 +1,166 @@ +#!/usr/bin/env python3 +# replays the test routes into the panda from the rack's UP and compares its safety decisions with libsafety's +import os +import sys +import time +from collections import Counter + +from opendbc.car.car_helpers import interfaces +from opendbc.car.tests.routes import routes +from opendbc.car.tests.test_models import NUM_JOBS, JOB_ID, TestCarModelBase +from opendbc.safety.tests.libsafety import libsafety_py + +SECONDS_PER_ROUTE = 10 +DT = 0.01 +BUSES = 3 +# seconds until the RX checks are compared +STEADY = 2 + + +def load(route): + R = type("R", (TestCarModelBase,), {"test_route": route, "platform": route.car_model}) + car_fw, can_msgs, alpha_long = R.get_testing_data() + CP = interfaces[R.platform].get_params(R.platform, R.fingerprint, car_fw, alpha_long, False, docs=False) + cfg = CP.safetyConfigs[-1] + + t0 = can_msgs[0][0] + frames = [[] for _ in range(int(SECONDS_PER_ROUTE / DT))] + for t, msgs in can_msgs: + i = int((t - t0) / 1e9 / DT) + if i >= len(frames): + break + frames[i] += [(m.address, bytes(m.dat), m.src % 4) for m in msgs if m.src < 64 and m.src % 4 < BUSES] + return cfg.safetyModel.raw, cfg.safetyParam, frames + + +def without_forwarded_copies(mode, param, frames): + # some logs also have the panda's forwarded copy, replaying it would collide with the panda's own forward + safety = libsafety_py.libsafety + assert safety.set_safety_hooks(mode, param) == 0 + out = [] + for frame in frames: + copies = Counter(frame) + drop = Counter() + for (addr, dat, bus), n in copies.items(): + if bus == 0 and (addr, dat, 2) in copies and safety.safety_fwd_hook(0, addr) == 2: + drop[(addr, dat, 2)] += min(n, copies[(addr, dat, 2)]) + elif bus == 2 and (addr, dat, 0) in copies and safety.safety_fwd_hook(2, addr) == 0 and safety.safety_fwd_hook(0, addr) != 2: + drop[(addr, dat, 0)] += min(n, copies[(addr, dat, 0)]) + kept = [] + for msg in frame: + if drop[msg]: + drop[msg] -= 1 + else: + kept.append(msg) + out.append(kept) + return out + + +def expected(mode, param, frames): + safety = libsafety_py.libsafety + assert safety.set_safety_hooks(mode, param) == 0 + safety.set_alternative_experience(0) + rx, fwd, invalid, checks_invalid = Counter(), Counter(), 0, False + for i, frame in enumerate(frames): + safety.set_timer(int(i * DT * 1e6)) + if i % int(1 / DT) == 0: + safety.safety_tick() + checks_invalid |= i * DT >= STEADY and bool(safety.safety_a.state.safety_rx_checks_invalid) + for addr, dat, bus in frame: + fwd[bus] += safety.safety_fwd_hook(bus, addr) >= 0 + invalid += not safety.safety_rx_hook(libsafety_py.make_CANPacket(addr, bus, dat)) + rx[bus] += 1 + return {"rx": [rx[b] for b in range(BUSES)], "fwd": [fwd[b] for b in range(BUSES)], "rx_invalid": invalid, + "relay_malfunction": bool(safety.get_relay_malfunction()), "rx_checks_invalid": checks_invalid, + "safety": [mode, param], "faults": 0, "tx_buffer_overflow": 0, "heartbeat_disabled": False, "harness_changed": False} + + +def within(lo, hi, got): + if isinstance(lo, list): + return all(within(a, b, g) for a, b, g in zip(lo, hi, got, strict=True)) + return min(lo, hi) <= got <= max(lo, hi) + + +def played(p, up, mode, param, frames): + # the safety mode first, it restarts the heartbeat timeout that would turn power save back on + p.set_safety_mode(mode, param) + # loading the route without a heartbeat puts the panda in power save + p.set_power_save(0) + # can_health is per CAN controller, a flipped harness swaps buses 0 and 2 + cans = [2, 1, 0] if p.health()["car_harness_status"] == p.HARNESS_STATUS_FLIPPED else [0, 1, 2] + hs0, h0 = p.health(), [p.can_health(c) for c in cans] + fd = {bus for frame in frames for _, dat, bus in frame if len(dat) > 8} + # the panda forwards CAN FD frames between buses 0 and 2 + if fd & {0, 2}: + fd |= {0, 2} + for b in fd: + up.set_can_data_speed_kbps(b, 2000) + # an empty tail, so the station's loop doesn't start over + up.can_replay(frames + [[]] * int(60 / DT)) + start, checks_invalid = time.monotonic(), False + while time.monotonic() < start + len(frames) * DT + 1: + p.send_heartbeat(engaged=False) + # the panda updates its RX checks once a second + if STEADY + 1 <= time.monotonic() - start <= len(frames) * DT: + checks_invalid |= p.health()["safety_rx_checks_invalid"] + time.sleep(0.2) + h, h1 = p.health(), [p.can_health(c) for c in cans] + up.reset() + up.set_ignition(True) + return {"rx": [h1[b]["total_rx_cnt"] - h0[b]["total_rx_cnt"] for b in range(BUSES)], + "fwd": [h1[b]["total_fwd_cnt"] - h0[b]["total_fwd_cnt"] for b in range(BUSES)], + "rx_invalid": h["safety_rx_invalid"], "relay_malfunction": bool(h["faults"] & 1), "rx_checks_invalid": checks_invalid, + "safety": [h["safety_mode"], h["safety_param"]], + "faults": h["faults"] & ~hs0["faults"] & ~1, + # only TX, the queue to the host overflows without pandad + "tx_buffer_overflow": h["tx_buffer_overflow"] - hs0["tx_buffer_overflow"], + "heartbeat_disabled": h["heartbeat_disabled"], "harness_changed": h["car_harness_status"] != hs0["car_harness_status"], + # only printed, to debug a failure + "can_errors": {k: [h1[b][k] - h0[b][k] for b in range(BUSES)] + for k in ("total_error_cnt", "total_tx_lost_cnt", "bus_off_cnt", "can_core_reset_count")}} + + +def main(): + from panda import Panda + from ultimate_provisioning import UP + + os.environ["SAFETY_UBSAN_TRAP"] = "1" + p, up = Panda(), UP.remote() + # the station starts with the harness lines floating + up.reset() + up.set_ignition(True) + + names = sys.argv[1:] + failed = [] + for i, route in enumerate(routes): + if i % NUM_JOBS != JOB_ID or (names and route.route not in names): + continue + st = time.monotonic() + try: + mode, param, frames = load(route) + frames = without_forwarded_copies(mode, param, frames) + except Exception as e: + print(f"{route.route} FAIL, can't load: {type(e).__name__}: {e!r}", flush=True) + failed.append(route.route) + continue + # the station doesn't control the order of one 10 ms list's frames on different buses + lo, hi = (expected(mode, param, [sorted(f, key=lambda m: m[2] * order) for f in frames]) for order in (1, -1)) + got = played(p, up, mode, param, frames) + # forwarding stops at a relay malfunction, when depends on the tick + if lo["relay_malfunction"] or hi["relay_malfunction"]: + for d in (lo, hi, got): + del d["fwd"] + ok = all(within(lo[k], hi[k], got[k]) for k in lo) + exp = lo if lo == hi else {"low": lo, "high": hi} + dropped = [e - g for e, g in zip(lo["rx"], got["rx"], strict=True)] + print(f"{route.route} {route.car_model} mode {mode} {'ok' if ok else 'FAIL'} {time.monotonic() - st:.0f}s expected {exp} panda {got}" + + (f" station dropped {dropped}" if any(dropped) else ""), flush=True) + if not ok: + failed.append(route.route) + + print(f"{len(failed)} failed: {failed}") + return int(bool(failed)) + + +if __name__ == "__main__": + sys.exit(main()) From 7dbc16b0a216ae2075fc2891b026f50b5fb26ab6 Mon Sep 17 00:00:00 2001 From: elkoled Date: Sat, 14 Nov 2026 13:51:52 -0800 Subject: [PATCH 02/21] 8 replay jobs, alfred allows 20 jobs --- alfred.toml | 28 +++++++++------------------- 1 file changed, 9 insertions(+), 19 deletions(-) diff --git a/alfred.toml b/alfred.toml index 6afdb227b9df1b..b37e55ec0b058f 100644 --- a/alfred.toml +++ b/alfred.toml @@ -125,55 +125,45 @@ uses = "build" name = "Sensord" run = "./openpilot/system/sensord/tests/test_sensord.py" -# a tenth of the safety replay routes each, on the up-can stations and any free comma four +# an eighth of the safety replay routes each, on the up-can stations and any free comma four [jobs.safety_replay_0] runs-on = "up-can" -env = { NUM_JOBS = "10", JOB_ID = "0" } +env = { NUM_JOBS = "8", JOB_ID = "0" } steps = [{ uses = "build" }, { uses = "flash-panda" }, { uses = "safety-replay" }] [jobs.safety_replay_1] runs-on = "up-can" -env = { NUM_JOBS = "10", JOB_ID = "1" } +env = { NUM_JOBS = "8", JOB_ID = "1" } steps = [{ uses = "build" }, { uses = "flash-panda" }, { uses = "safety-replay" }] [jobs.safety_replay_2] runs-on = "up-can" -env = { NUM_JOBS = "10", JOB_ID = "2" } +env = { NUM_JOBS = "8", JOB_ID = "2" } steps = [{ uses = "build" }, { uses = "flash-panda" }, { uses = "safety-replay" }] [jobs.safety_replay_3] runs-on = "up-can" -env = { NUM_JOBS = "10", JOB_ID = "3" } +env = { NUM_JOBS = "8", JOB_ID = "3" } steps = [{ uses = "build" }, { uses = "flash-panda" }, { uses = "safety-replay" }] [jobs.safety_replay_4] runs-on = "up-can" -env = { NUM_JOBS = "10", JOB_ID = "4" } +env = { NUM_JOBS = "8", JOB_ID = "4" } steps = [{ uses = "build" }, { uses = "flash-panda" }, { uses = "safety-replay" }] [jobs.safety_replay_5] runs-on = "up-can" -env = { NUM_JOBS = "10", JOB_ID = "5" } +env = { NUM_JOBS = "8", JOB_ID = "5" } steps = [{ uses = "build" }, { uses = "flash-panda" }, { uses = "safety-replay" }] [jobs.safety_replay_6] runs-on = "up-can" -env = { NUM_JOBS = "10", JOB_ID = "6" } +env = { NUM_JOBS = "8", JOB_ID = "6" } steps = [{ uses = "build" }, { uses = "flash-panda" }, { uses = "safety-replay" }] [jobs.safety_replay_7] runs-on = "up-can" -env = { NUM_JOBS = "10", JOB_ID = "7" } -steps = [{ uses = "build" }, { uses = "flash-panda" }, { uses = "safety-replay" }] - -[jobs.safety_replay_8] -runs-on = "up-can" -env = { NUM_JOBS = "10", JOB_ID = "8" } -steps = [{ uses = "build" }, { uses = "flash-panda" }, { uses = "safety-replay" }] - -[jobs.safety_replay_9] -runs-on = "up-can" -env = { NUM_JOBS = "10", JOB_ID = "9" } +env = { NUM_JOBS = "8", JOB_ID = "7" } steps = [{ uses = "build" }, { uses = "flash-panda" }, { uses = "safety-replay" }] # a comma four with a Chestnut on its aux port, which runs the big model From 885f133ea71f99fc522214d3eab1a60922f25a13 Mon Sep 17 00:00:00 2001 From: elkoled Date: Sun, 15 Nov 2026 23:48:43 -0800 Subject: [PATCH 03/21] bump opendbc --- alfred.toml | 2 +- opendbc_repo | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/alfred.toml b/alfred.toml index b37e55ec0b058f..d4788d9819c2b2 100644 --- a/alfred.toml +++ b/alfred.toml @@ -1,5 +1,5 @@ # a UP's DUT starts each job with an empty /data: these come back from master's last good job, as it left them -cache = ["/data/scons_cache", "/data/tmp/comma_download_cache"] +cache = ["/data/scons_cache"] [env] PYTHONPATH = "/data/ci/tree/openpilot" # the tree diff --git a/opendbc_repo b/opendbc_repo index 4b0d18fd0f8916..a58d98496de0a6 160000 --- a/opendbc_repo +++ b/opendbc_repo @@ -1 +1 @@ -Subproject commit 4b0d18fd0f8916ba61101f960e97515a7ae4622f +Subproject commit a58d98496de0a6e6db65603fe0207094fa721490 From fb355de8c4c1178e594f9ccf302435749a5e7291 Mon Sep 17 00:00:00 2001 From: elkoled Date: Sun, 15 Nov 2026 23:51:49 -0800 Subject: [PATCH 04/21] clearer comments --- alfred.toml | 2 +- .../selfdrive/pandad/tests/safety_replay.py | 20 +++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/alfred.toml b/alfred.toml index d4788d9819c2b2..3d32ced02e9c84 100644 --- a/alfred.toml +++ b/alfred.toml @@ -125,7 +125,7 @@ uses = "build" name = "Sensord" run = "./openpilot/system/sensord/tests/test_sensord.py" -# an eighth of the safety replay routes each, on the up-can stations and any free comma four +# each job replays an eighth of the routes [jobs.safety_replay_0] runs-on = "up-can" env = { NUM_JOBS = "8", JOB_ID = "0" } diff --git a/openpilot/selfdrive/pandad/tests/safety_replay.py b/openpilot/selfdrive/pandad/tests/safety_replay.py index 6b1c713bd7ed68..20bc29ffcfa7c2 100755 --- a/openpilot/selfdrive/pandad/tests/safety_replay.py +++ b/openpilot/selfdrive/pandad/tests/safety_replay.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# replays the test routes into the panda from the rack's UP and compares its safety decisions with libsafety's +# checks the panda's safety against libsafety on the car test routes import os import sys import time @@ -34,7 +34,7 @@ def load(route): def without_forwarded_copies(mode, param, frames): - # some logs also have the panda's forwarded copy, replaying it would collide with the panda's own forward + # drop the panda's forwarded copies in the log, the panda forwards them again safety = libsafety_py.libsafety assert safety.set_safety_hooks(mode, param) == 0 out = [] @@ -82,9 +82,9 @@ def within(lo, hi, got): def played(p, up, mode, param, frames): - # the safety mode first, it restarts the heartbeat timeout that would turn power save back on + # first, it resets the heartbeat timeout that turns power save back on p.set_safety_mode(mode, param) - # loading the route without a heartbeat puts the panda in power save + # the panda went into power save while the route loaded without a heartbeat p.set_power_save(0) # can_health is per CAN controller, a flipped harness swaps buses 0 and 2 cans = [2, 1, 0] if p.health()["car_harness_status"] == p.HARNESS_STATUS_FLIPPED else [0, 1, 2] @@ -95,7 +95,7 @@ def played(p, up, mode, param, frames): fd |= {0, 2} for b in fd: up.set_can_data_speed_kbps(b, 2000) - # an empty tail, so the station's loop doesn't start over + # pad the end so the UP doesn't loop the route up.can_replay(frames + [[]] * int(60 / DT)) start, checks_invalid = time.monotonic(), False while time.monotonic() < start + len(frames) * DT + 1: @@ -112,7 +112,7 @@ def played(p, up, mode, param, frames): "rx_invalid": h["safety_rx_invalid"], "relay_malfunction": bool(h["faults"] & 1), "rx_checks_invalid": checks_invalid, "safety": [h["safety_mode"], h["safety_param"]], "faults": h["faults"] & ~hs0["faults"] & ~1, - # only TX, the queue to the host overflows without pandad + # TX only, RX overflows without pandad "tx_buffer_overflow": h["tx_buffer_overflow"] - hs0["tx_buffer_overflow"], "heartbeat_disabled": h["heartbeat_disabled"], "harness_changed": h["car_harness_status"] != hs0["car_harness_status"], # only printed, to debug a failure @@ -126,7 +126,7 @@ def main(): os.environ["SAFETY_UBSAN_TRAP"] = "1" p, up = Panda(), UP.remote() - # the station starts with the harness lines floating + # the UP starts with the harness lines floating up.reset() up.set_ignition(True) @@ -143,10 +143,10 @@ def main(): print(f"{route.route} FAIL, can't load: {type(e).__name__}: {e!r}", flush=True) failed.append(route.route) continue - # the station doesn't control the order of one 10 ms list's frames on different buses + # the UP sends a 10 ms step's frames on different buses in any order lo, hi = (expected(mode, param, [sorted(f, key=lambda m: m[2] * order) for f in frames]) for order in (1, -1)) got = played(p, up, mode, param, frames) - # forwarding stops at a relay malfunction, when depends on the tick + # forwarding stops at a relay malfunction, exactly when depends on the tick if lo["relay_malfunction"] or hi["relay_malfunction"]: for d in (lo, hi, got): del d["fwd"] @@ -154,7 +154,7 @@ def main(): exp = lo if lo == hi else {"low": lo, "high": hi} dropped = [e - g for e, g in zip(lo["rx"], got["rx"], strict=True)] print(f"{route.route} {route.car_model} mode {mode} {'ok' if ok else 'FAIL'} {time.monotonic() - st:.0f}s expected {exp} panda {got}" + - (f" station dropped {dropped}" if any(dropped) else ""), flush=True) + (f" UP dropped {dropped}" if any(dropped) else ""), flush=True) if not ok: failed.append(route.route) From f610e65b598d60df528d7fa3acf15cfa9e36d555 Mon Sep 17 00:00:00 2001 From: elkoled Date: Mon, 16 Nov 2026 06:31:23 -0800 Subject: [PATCH 05/21] safety replay: retry route download --- openpilot/selfdrive/pandad/tests/safety_replay.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/openpilot/selfdrive/pandad/tests/safety_replay.py b/openpilot/selfdrive/pandad/tests/safety_replay.py index 20bc29ffcfa7c2..15ad40a33394e0 100755 --- a/openpilot/selfdrive/pandad/tests/safety_replay.py +++ b/openpilot/selfdrive/pandad/tests/safety_replay.py @@ -9,6 +9,7 @@ from opendbc.car.tests.routes import routes from opendbc.car.tests.test_models import NUM_JOBS, JOB_ID, TestCarModelBase from opendbc.safety.tests.libsafety import libsafety_py +from openpilot.common.utils import retry SECONDS_PER_ROUTE = 10 DT = 0.01 @@ -17,6 +18,8 @@ STEADY = 2 +# a route download can drop mid file +@retry(attempts=3, delay=10) def load(route): R = type("R", (TestCarModelBase,), {"test_route": route, "platform": route.car_model}) car_fw, can_msgs, alpha_long = R.get_testing_data() From 81b1f6047dae0f60c3ca4b8fa7257077e4425dbd Mon Sep 17 00:00:00 2001 From: elkoled Date: Mon, 16 Nov 2026 10:18:36 -0800 Subject: [PATCH 06/21] safety replay: shorter job names --- alfred.toml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/alfred.toml b/alfred.toml index 3d32ced02e9c84..8cc27e340d0ee8 100644 --- a/alfred.toml +++ b/alfred.toml @@ -126,42 +126,42 @@ name = "Sensord" run = "./openpilot/system/sensord/tests/test_sensord.py" # each job replays an eighth of the routes -[jobs.safety_replay_0] +[jobs.safety_0] runs-on = "up-can" env = { NUM_JOBS = "8", JOB_ID = "0" } steps = [{ uses = "build" }, { uses = "flash-panda" }, { uses = "safety-replay" }] -[jobs.safety_replay_1] +[jobs.safety_1] runs-on = "up-can" env = { NUM_JOBS = "8", JOB_ID = "1" } steps = [{ uses = "build" }, { uses = "flash-panda" }, { uses = "safety-replay" }] -[jobs.safety_replay_2] +[jobs.safety_2] runs-on = "up-can" env = { NUM_JOBS = "8", JOB_ID = "2" } steps = [{ uses = "build" }, { uses = "flash-panda" }, { uses = "safety-replay" }] -[jobs.safety_replay_3] +[jobs.safety_3] runs-on = "up-can" env = { NUM_JOBS = "8", JOB_ID = "3" } steps = [{ uses = "build" }, { uses = "flash-panda" }, { uses = "safety-replay" }] -[jobs.safety_replay_4] +[jobs.safety_4] runs-on = "up-can" env = { NUM_JOBS = "8", JOB_ID = "4" } steps = [{ uses = "build" }, { uses = "flash-panda" }, { uses = "safety-replay" }] -[jobs.safety_replay_5] +[jobs.safety_5] runs-on = "up-can" env = { NUM_JOBS = "8", JOB_ID = "5" } steps = [{ uses = "build" }, { uses = "flash-panda" }, { uses = "safety-replay" }] -[jobs.safety_replay_6] +[jobs.safety_6] runs-on = "up-can" env = { NUM_JOBS = "8", JOB_ID = "6" } steps = [{ uses = "build" }, { uses = "flash-panda" }, { uses = "safety-replay" }] -[jobs.safety_replay_7] +[jobs.safety_7] runs-on = "up-can" env = { NUM_JOBS = "8", JOB_ID = "7" } steps = [{ uses = "build" }, { uses = "flash-panda" }, { uses = "safety-replay" }] From 3f206b840fd7aaad5b6dd3ced041624df55cd175 Mon Sep 17 00:00:00 2001 From: elkoled Date: Mon, 16 Nov 2026 16:01:56 -0800 Subject: [PATCH 07/21] bump opendbc and panda --- opendbc_repo | 2 +- openpilot/selfdrive/pandad/tests/safety_replay.py | 2 +- panda | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/opendbc_repo b/opendbc_repo index a58d98496de0a6..ec18ed089a32e4 160000 --- a/opendbc_repo +++ b/opendbc_repo @@ -1 +1 @@ -Subproject commit a58d98496de0a6e6db65603fe0207094fa721490 +Subproject commit ec18ed089a32e4bdc7e259e1f2d0725baf73a48a diff --git a/openpilot/selfdrive/pandad/tests/safety_replay.py b/openpilot/selfdrive/pandad/tests/safety_replay.py index 15ad40a33394e0..24d4e4cb72afd7 100755 --- a/openpilot/selfdrive/pandad/tests/safety_replay.py +++ b/openpilot/selfdrive/pandad/tests/safety_replay.py @@ -22,7 +22,7 @@ @retry(attempts=3, delay=10) def load(route): R = type("R", (TestCarModelBase,), {"test_route": route, "platform": route.car_model}) - car_fw, can_msgs, alpha_long = R.get_testing_data() + car_fw, can_msgs, alpha_long = R.get_testing_data(route) CP = interfaces[R.platform].get_params(R.platform, R.fingerprint, car_fw, alpha_long, False, docs=False) cfg = CP.safetyConfigs[-1] diff --git a/panda b/panda index bf14f329132082..8ae88ad1d93096 160000 --- a/panda +++ b/panda @@ -1 +1 @@ -Subproject commit bf14f329132082db7559ddca33598be7c6272528 +Subproject commit 8ae88ad1d93096c64e3e9fc143853ad7bd9ddac9 From a9b989d120051ec9f38f8338b5081ec9dfef6068 Mon Sep 17 00:00:00 2001 From: elkoled Date: Mon, 16 Nov 2026 16:07:52 -0800 Subject: [PATCH 08/21] safety replay: prepare the next route while one replays, job timeouts --- alfred.toml | 8 ++++++ .../selfdrive/pandad/tests/safety_replay.py | 26 ++++++++++++++----- 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/alfred.toml b/alfred.toml index 8cc27e340d0ee8..f6c880f4088902 100644 --- a/alfred.toml +++ b/alfred.toml @@ -128,41 +128,49 @@ run = "./openpilot/system/sensord/tests/test_sensord.py" # each job replays an eighth of the routes [jobs.safety_0] runs-on = "up-can" +timeout-minutes = 25 env = { NUM_JOBS = "8", JOB_ID = "0" } steps = [{ uses = "build" }, { uses = "flash-panda" }, { uses = "safety-replay" }] [jobs.safety_1] runs-on = "up-can" +timeout-minutes = 25 env = { NUM_JOBS = "8", JOB_ID = "1" } steps = [{ uses = "build" }, { uses = "flash-panda" }, { uses = "safety-replay" }] [jobs.safety_2] runs-on = "up-can" +timeout-minutes = 25 env = { NUM_JOBS = "8", JOB_ID = "2" } steps = [{ uses = "build" }, { uses = "flash-panda" }, { uses = "safety-replay" }] [jobs.safety_3] runs-on = "up-can" +timeout-minutes = 25 env = { NUM_JOBS = "8", JOB_ID = "3" } steps = [{ uses = "build" }, { uses = "flash-panda" }, { uses = "safety-replay" }] [jobs.safety_4] runs-on = "up-can" +timeout-minutes = 25 env = { NUM_JOBS = "8", JOB_ID = "4" } steps = [{ uses = "build" }, { uses = "flash-panda" }, { uses = "safety-replay" }] [jobs.safety_5] runs-on = "up-can" +timeout-minutes = 25 env = { NUM_JOBS = "8", JOB_ID = "5" } steps = [{ uses = "build" }, { uses = "flash-panda" }, { uses = "safety-replay" }] [jobs.safety_6] runs-on = "up-can" +timeout-minutes = 25 env = { NUM_JOBS = "8", JOB_ID = "6" } steps = [{ uses = "build" }, { uses = "flash-panda" }, { uses = "safety-replay" }] [jobs.safety_7] runs-on = "up-can" +timeout-minutes = 25 env = { NUM_JOBS = "8", JOB_ID = "7" } steps = [{ uses = "build" }, { uses = "flash-panda" }, { uses = "safety-replay" }] diff --git a/openpilot/selfdrive/pandad/tests/safety_replay.py b/openpilot/selfdrive/pandad/tests/safety_replay.py index 24d4e4cb72afd7..9380d9798d02e0 100755 --- a/openpilot/selfdrive/pandad/tests/safety_replay.py +++ b/openpilot/selfdrive/pandad/tests/safety_replay.py @@ -1,9 +1,11 @@ #!/usr/bin/env python3 # checks the panda's safety against libsafety on the car test routes +import multiprocessing import os import sys import time from collections import Counter +from concurrent.futures import ProcessPoolExecutor from opendbc.car.car_helpers import interfaces from opendbc.car.tests.routes import routes @@ -78,6 +80,14 @@ def expected(mode, param, frames): "safety": [mode, param], "faults": 0, "tx_buffer_overflow": 0, "heartbeat_disabled": False, "harness_changed": False} +def prepare(route): + mode, param, frames = load(route) + frames = without_forwarded_copies(mode, param, frames) + # the UP sends a 10 ms step's frames on different buses in any order + lo, hi = (expected(mode, param, [sorted(f, key=lambda m: m[2] * order) for f in frames]) for order in (1, -1)) + return mode, param, frames, lo, hi + + def within(lo, hi, got): if isinstance(lo, list): return all(within(a, b, g) for a, b, g in zip(lo, hi, got, strict=True)) @@ -134,20 +144,21 @@ def main(): up.set_ignition(True) names = sys.argv[1:] + todo = [r for i, r in enumerate(routes) if i % NUM_JOBS == JOB_ID and (not names or r.route in names)] failed = [] - for i, route in enumerate(routes): - if i % NUM_JOBS != JOB_ID or (names and route.route not in names): - continue + # loads the next route while this one replays + pool = ProcessPoolExecutor(1, mp_context=multiprocessing.get_context("spawn")) + loading = pool.submit(prepare, todo[0]) if todo else None + for n, route in enumerate(todo): st = time.monotonic() + current = loading + loading = pool.submit(prepare, todo[n + 1]) if n + 1 < len(todo) else None try: - mode, param, frames = load(route) - frames = without_forwarded_copies(mode, param, frames) + mode, param, frames, lo, hi = current.result() except Exception as e: print(f"{route.route} FAIL, can't load: {type(e).__name__}: {e!r}", flush=True) failed.append(route.route) continue - # the UP sends a 10 ms step's frames on different buses in any order - lo, hi = (expected(mode, param, [sorted(f, key=lambda m: m[2] * order) for f in frames]) for order in (1, -1)) got = played(p, up, mode, param, frames) # forwarding stops at a relay malfunction, exactly when depends on the tick if lo["relay_malfunction"] or hi["relay_malfunction"]: @@ -161,6 +172,7 @@ def main(): if not ok: failed.append(route.route) + pool.shutdown() print(f"{len(failed)} failed: {failed}") return int(bool(failed)) From 3a03bc4268957e922b23376197c3fcaa64f35b1e Mon Sep 17 00:00:00 2001 From: elkoled Date: Mon, 16 Nov 2026 16:31:42 -0800 Subject: [PATCH 09/21] safety replay: skip inputs master passed, compare safety with the previous opendbc --- .github/workflows/tests.yaml | 27 ++++ alfred.toml | 22 ++- opendbc_repo | 2 +- .../selfdrive/pandad/tests/safety_diff.sh | 27 ++++ .../selfdrive/pandad/tests/safety_replay.py | 127 ++++++++++++------ 5 files changed, 151 insertions(+), 54 deletions(-) create mode 100755 openpilot/selfdrive/pandad/tests/safety_diff.sh diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 4777bc9c427c30..73ae7ad004c209 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -174,6 +174,33 @@ jobs: max_attempts: 3 command: cd ${{ github.workspace }}/ci-artifacts && git push origin process-replay --force + safety_replay: + name: safety replay + runs-on: ${{ + (github.repository == 'commaai/openpilot') && + ((github.event_name != 'pull_request') || + (github.event.pull_request.head.repo.full_name == 'commaai/openpilot')) + && fromJSON('["namespace-profile-amd64-8x16"]') + || fromJSON('["ubuntu-24.04"]') }} + env: + COMMA_CACHE: /tmp/comma_download_cache + steps: + - uses: actions/checkout@v7 + with: + submodules: true + # the parent is master, or the previous master commit on a push + fetch-depth: 2 + - run: ./tools/op.sh setup + - name: Cache the test routes + uses: actions/cache@v6 + with: + path: /tmp/comma_download_cache + key: safety-replay-${{ hashFiles('opendbc_repo/opendbc/car/tests/routes.py') }} + restore-keys: safety-replay- + - name: Compare safety with the previous opendbc + timeout-minutes: 20 + run: openpilot/selfdrive/pandad/tests/safety_diff.sh HEAD^ + simulator_driving: name: simulator driving runs-on: ${{ diff --git a/alfred.toml b/alfred.toml index f6c880f4088902..3b96e10a08c730 100644 --- a/alfred.toml +++ b/alfred.toml @@ -23,10 +23,6 @@ for p in msgq_repo/msgq opendbc_repo/opendbc rednose_repo/rednose teleoprtc_repo tools/op.sh build """ -[steps.flash-panda] -name = "Flash panda" -run = "python3 -c 'from openpilot.selfdrive.pandad.pandad import flash_panda; flash_panda()'" - [steps.safety-replay] name = "Safety replay" run = "./openpilot/selfdrive/pandad/tests/safety_replay.py" @@ -125,54 +121,54 @@ uses = "build" name = "Sensord" run = "./openpilot/system/sensord/tests/test_sensord.py" -# each job replays an eighth of the routes +# each job replays an eighth of the routes, unless master passed the same inputs in the last day [jobs.safety_0] runs-on = "up-can" timeout-minutes = 25 env = { NUM_JOBS = "8", JOB_ID = "0" } -steps = [{ uses = "build" }, { uses = "flash-panda" }, { uses = "safety-replay" }] +steps = [{ uses = "build" }, { uses = "safety-replay" }] [jobs.safety_1] runs-on = "up-can" timeout-minutes = 25 env = { NUM_JOBS = "8", JOB_ID = "1" } -steps = [{ uses = "build" }, { uses = "flash-panda" }, { uses = "safety-replay" }] +steps = [{ uses = "build" }, { uses = "safety-replay" }] [jobs.safety_2] runs-on = "up-can" timeout-minutes = 25 env = { NUM_JOBS = "8", JOB_ID = "2" } -steps = [{ uses = "build" }, { uses = "flash-panda" }, { uses = "safety-replay" }] +steps = [{ uses = "build" }, { uses = "safety-replay" }] [jobs.safety_3] runs-on = "up-can" timeout-minutes = 25 env = { NUM_JOBS = "8", JOB_ID = "3" } -steps = [{ uses = "build" }, { uses = "flash-panda" }, { uses = "safety-replay" }] +steps = [{ uses = "build" }, { uses = "safety-replay" }] [jobs.safety_4] runs-on = "up-can" timeout-minutes = 25 env = { NUM_JOBS = "8", JOB_ID = "4" } -steps = [{ uses = "build" }, { uses = "flash-panda" }, { uses = "safety-replay" }] +steps = [{ uses = "build" }, { uses = "safety-replay" }] [jobs.safety_5] runs-on = "up-can" timeout-minutes = 25 env = { NUM_JOBS = "8", JOB_ID = "5" } -steps = [{ uses = "build" }, { uses = "flash-panda" }, { uses = "safety-replay" }] +steps = [{ uses = "build" }, { uses = "safety-replay" }] [jobs.safety_6] runs-on = "up-can" timeout-minutes = 25 env = { NUM_JOBS = "8", JOB_ID = "6" } -steps = [{ uses = "build" }, { uses = "flash-panda" }, { uses = "safety-replay" }] +steps = [{ uses = "build" }, { uses = "safety-replay" }] [jobs.safety_7] runs-on = "up-can" timeout-minutes = 25 env = { NUM_JOBS = "8", JOB_ID = "7" } -steps = [{ uses = "build" }, { uses = "flash-panda" }, { uses = "safety-replay" }] +steps = [{ uses = "build" }, { uses = "safety-replay" }] # a comma four with a Chestnut on its aux port, which runs the big model [jobs.chestnut] diff --git a/opendbc_repo b/opendbc_repo index ec18ed089a32e4..533372f1299a8f 160000 --- a/opendbc_repo +++ b/opendbc_repo @@ -1 +1 @@ -Subproject commit ec18ed089a32e4bdc7e259e1f2d0725baf73a48a +Subproject commit 533372f1299a8f6c9da98623152e2e862197d3fc diff --git a/openpilot/selfdrive/pandad/tests/safety_diff.sh b/openpilot/selfdrive/pandad/tests/safety_diff.sh new file mode 100755 index 00000000000000..0dd354c449c1b5 --- /dev/null +++ b/openpilot/selfdrive/pandad/tests/safety_diff.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# compares every safety RX and TX result and the safety state on the car test routes, between BASE's opendbc and this one +set -e + +cd "$(dirname "$0")/../../../.." +BASE=${1:-HEAD^} + +base=$(git rev-parse "$BASE:opendbc_repo") +new=$(git rev-parse HEAD:opendbc_repo) +if [ "$base" = "$new" ]; then + echo "opendbc is unchanged ($new), nothing to compare" + exit 0 +fi +echo "comparing opendbc $base -> $new" + +W=$(mktemp -d) +trap 'git -C opendbc_repo worktree remove --force "$W/base" 2>/dev/null; rm -rf "$W"' EXIT +git -C opendbc_repo fetch -q --depth=1 origin "$base" +git -C opendbc_repo worktree add -q --detach "$W/base" "$base" +# the base's safety code, built like this opendbc builds libsafety +cp opendbc_repo/opendbc/safety/tests/libsafety/libsafety_py.py "$W/base/opendbc/safety/tests/libsafety/" + +REPLAY=opendbc_repo/opendbc/safety/tests/safety_replay/route_replay.py +python3 $REPLAY dump "$W/routes" +PYTHONPATH="$W/base" python3 $REPLAY digest "$W/routes" "$W/base.json" +python3 $REPLAY digest "$W/routes" "$W/new.json" +python3 $REPLAY compare "$W/base.json" "$W/new.json" diff --git a/openpilot/selfdrive/pandad/tests/safety_replay.py b/openpilot/selfdrive/pandad/tests/safety_replay.py index 9380d9798d02e0..686eb14c968e55 100755 --- a/openpilot/selfdrive/pandad/tests/safety_replay.py +++ b/openpilot/selfdrive/pandad/tests/safety_replay.py @@ -1,41 +1,62 @@ #!/usr/bin/env python3 # checks the panda's safety against libsafety on the car test routes +import hashlib import multiprocessing import os +import subprocess import sys import time from collections import Counter from concurrent.futures import ProcessPoolExecutor +from concurrent.futures.process import BrokenProcessPool +from pathlib import Path -from opendbc.car.car_helpers import interfaces from opendbc.car.tests.routes import routes -from opendbc.car.tests.test_models import NUM_JOBS, JOB_ID, TestCarModelBase +from opendbc.car.tests.test_models import NUM_JOBS, JOB_ID from opendbc.safety.tests.libsafety import libsafety_py +from opendbc.safety.tests.safety_replay.route_replay import BUSES, DT, load, replay +from openpilot.common.basedir import BASEDIR +from openpilot.common.hardware import HARDWARE from openpilot.common.utils import retry SECONDS_PER_ROUTE = 10 -DT = 0.01 -BUSES = 3 # seconds until the RX checks are compared STEADY = 2 +# master's passing runs, restored with the alfred cache +PASSED = Path("/data/scons_cache/safety_replay") +# a pass is reused for a day, so the rack itself is still checked daily +PASS_TTL = 24 * 3600 +BUILD_OUTPUTS = {".so", ".o", ".os", ".a", ".pyc", ".gcno", ".gcda"} + + +def inputs() -> str: + # everything the result depends on, except the rack: the firmware, opendbc, the panda library, this test, the OS and its compiler + root = Path(BASEDIR) + files = sorted((root / "panda/board/obj").glob("*.bin.signed")) + for d in ("opendbc_repo/opendbc", "panda/python"): + files += sorted(f for f in (root / d).rglob("*") if f.is_file() and f.suffix not in BUILD_OUTPUTS and "__pycache__" not in f.parts) + files.append(Path(__file__).resolve()) + h = hashlib.sha256() + for f in files: + h.update(str(f.relative_to(root)).encode() + b"\0" + f.read_bytes()) + h.update(f"{HARDWARE.get_os_version()}".encode() + subprocess.check_output(["cc", "--version"])) + h.update(f"{NUM_JOBS} {JOB_ID}".encode()) + return h.hexdigest()[:16] + + +def passed_age(key: str) -> float | None: + try: + # wall clock, the pass was written on another DUT + age = time.time() - float((PASSED / key).read_text()) # noqa: TID251 + except (OSError, ValueError): + return None + return age if age < PASS_TTL else None # a route download can drop mid file @retry(attempts=3, delay=10) -def load(route): - R = type("R", (TestCarModelBase,), {"test_route": route, "platform": route.car_model}) - car_fw, can_msgs, alpha_long = R.get_testing_data(route) - CP = interfaces[R.platform].get_params(R.platform, R.fingerprint, car_fw, alpha_long, False, docs=False) - cfg = CP.safetyConfigs[-1] - - t0 = can_msgs[0][0] - frames = [[] for _ in range(int(SECONDS_PER_ROUTE / DT))] - for t, msgs in can_msgs: - i = int((t - t0) / 1e9 / DT) - if i >= len(frames): - break - frames[i] += [(m.address, bytes(m.dat), m.src % 4) for m in msgs if m.src < 64 and m.src % 4 < BUSES] - return cfg.safetyModel.raw, cfg.safetyParam, frames +def load_window(route): + return load(route, SECONDS_PER_ROUTE) def without_forwarded_copies(mode, param, frames): @@ -62,32 +83,35 @@ def without_forwarded_copies(mode, param, frames): def expected(mode, param, frames): - safety = libsafety_py.libsafety - assert safety.set_safety_hooks(mode, param) == 0 - safety.set_alternative_experience(0) - rx, fwd, invalid, checks_invalid = Counter(), Counter(), 0, False - for i, frame in enumerate(frames): - safety.set_timer(int(i * DT * 1e6)) - if i % int(1 / DT) == 0: - safety.safety_tick() - checks_invalid |= i * DT >= STEADY and bool(safety.safety_a.state.safety_rx_checks_invalid) - for addr, dat, bus in frame: - fwd[bus] += safety.safety_fwd_hook(bus, addr) >= 0 - invalid += not safety.safety_rx_hook(libsafety_py.make_CANPacket(addr, bus, dat)) + checks_invalid = [] + + def on_tick(i, safety): + checks_invalid.append(i * DT >= STEADY and bool(safety.safety_a.state.safety_rx_checks_invalid)) + + steps = replay(mode, param, frames, on_tick) + rx, fwd, invalid = Counter(), Counter(), 0 + for frame, (results, _) in zip(frames, steps, strict=True): + for (_, _, bus), (valid, destination) in zip(frame, results, strict=True): rx[bus] += 1 + fwd[bus] += destination >= 0 + invalid += not valid return {"rx": [rx[b] for b in range(BUSES)], "fwd": [fwd[b] for b in range(BUSES)], "rx_invalid": invalid, - "relay_malfunction": bool(safety.get_relay_malfunction()), "rx_checks_invalid": checks_invalid, + "relay_malfunction": bool(libsafety_py.libsafety.get_relay_malfunction()), "rx_checks_invalid": any(checks_invalid), "safety": [mode, param], "faults": 0, "tx_buffer_overflow": 0, "heartbeat_disabled": False, "harness_changed": False} def prepare(route): - mode, param, frames = load(route) + mode, param, frames = load_window(route) frames = without_forwarded_copies(mode, param, frames) # the UP sends a 10 ms step's frames on different buses in any order lo, hi = (expected(mode, param, [sorted(f, key=lambda m: m[2] * order) for f in frames]) for order in (1, -1)) return mode, param, frames, lo, hi +def new_pool(): + return ProcessPoolExecutor(1, mp_context=multiprocessing.get_context("spawn")) + + def within(lo, hi, got): if isinstance(lo, list): return all(within(a, b, g) for a, b, g in zip(lo, hi, got, strict=True)) @@ -136,7 +160,19 @@ def played(p, up, mode, param, frames): def main(): from panda import Panda from ultimate_provisioning import UP - + from openpilot.selfdrive.pandad.pandad import flash_panda + + key = inputs() + age = passed_age(key) + if age is None: + print(f"inputs {key} have no master pass in the last {PASS_TTL // 3600} h, replaying", flush=True) + elif os.environ.get("SAFETY_REPLAY_FORCE") == "1": + print(f"inputs {key} passed on master {age / 3600:.1f} h ago, replaying anyway (SAFETY_REPLAY_FORCE)", flush=True) + else: + print(f"skipped: master passed these inputs ({key}) {age / 3600:.1f} h ago, same firmware, opendbc, panda library, test and OS") + return 0 + + flash_panda() os.environ["SAFETY_UBSAN_TRAP"] = "1" p, up = Panda(), UP.remote() # the UP starts with the harness lines floating @@ -146,19 +182,26 @@ def main(): names = sys.argv[1:] todo = [r for i, r in enumerate(routes) if i % NUM_JOBS == JOB_ID and (not names or r.route in names)] failed = [] - # loads the next route while this one replays - pool = ProcessPoolExecutor(1, mp_context=multiprocessing.get_context("spawn")) + # prepares the next route while this one replays + pool = new_pool() loading = pool.submit(prepare, todo[0]) if todo else None for n, route in enumerate(todo): st = time.monotonic() - current = loading - loading = pool.submit(prepare, todo[n + 1]) if n + 1 < len(todo) else None try: - mode, param, frames, lo, hi = current.result() + prepared = loading.result() + except BrokenProcessPool as e: + # a crash in libsafety, like a UBSan trap + prepared = e + pool = new_pool() except Exception as e: - print(f"{route.route} FAIL, can't load: {type(e).__name__}: {e!r}", flush=True) + prepared = e + if n + 1 < len(todo): + loading = pool.submit(prepare, todo[n + 1]) + if isinstance(prepared, Exception): + print(f"{route.route} FAIL, can't prepare: {type(prepared).__name__}: {prepared!r}", flush=True) failed.append(route.route) continue + mode, param, frames, lo, hi = prepared got = played(p, up, mode, param, frames) # forwarding stops at a relay malfunction, exactly when depends on the tick if lo["relay_malfunction"] or hi["relay_malfunction"]: @@ -174,7 +217,11 @@ def main(): pool.shutdown() print(f"{len(failed)} failed: {failed}") - return int(bool(failed)) + if failed or names: + return int(bool(failed)) + PASSED.mkdir(parents=True, exist_ok=True) + (PASSED / key).write_text(str(time.time())) # noqa: TID251 + return 0 if __name__ == "__main__": From 55bdaab49e984f573ba7e2195b51941b382b3dd7 Mon Sep 17 00:00:00 2001 From: elkoled Date: Mon, 16 Nov 2026 16:35:01 -0800 Subject: [PATCH 10/21] safety diff: quote expansions --- openpilot/selfdrive/pandad/tests/safety_diff.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/openpilot/selfdrive/pandad/tests/safety_diff.sh b/openpilot/selfdrive/pandad/tests/safety_diff.sh index 0dd354c449c1b5..d50151a3fb0518 100755 --- a/openpilot/selfdrive/pandad/tests/safety_diff.sh +++ b/openpilot/selfdrive/pandad/tests/safety_diff.sh @@ -21,7 +21,7 @@ git -C opendbc_repo worktree add -q --detach "$W/base" "$base" cp opendbc_repo/opendbc/safety/tests/libsafety/libsafety_py.py "$W/base/opendbc/safety/tests/libsafety/" REPLAY=opendbc_repo/opendbc/safety/tests/safety_replay/route_replay.py -python3 $REPLAY dump "$W/routes" -PYTHONPATH="$W/base" python3 $REPLAY digest "$W/routes" "$W/base.json" -python3 $REPLAY digest "$W/routes" "$W/new.json" -python3 $REPLAY compare "$W/base.json" "$W/new.json" +python3 "$REPLAY" dump "$W/routes" +PYTHONPATH="$W/base" python3 "$REPLAY" digest "$W/routes" "$W/base.json" +python3 "$REPLAY" digest "$W/routes" "$W/new.json" +python3 "$REPLAY" compare "$W/base.json" "$W/new.json" From cb8803c88f10c97922aa4462e22c7b5d96e317a7 Mon Sep 17 00:00:00 2001 From: elkoled Date: Mon, 16 Nov 2026 16:50:13 -0800 Subject: [PATCH 11/21] safety replay: compare safety in openpilot, opendbc master --- .github/workflows/tests.yaml | 2 +- opendbc_repo | 2 +- .../selfdrive/pandad/tests/safety_diff.py | 157 ++++++++++++++++++ .../selfdrive/pandad/tests/safety_diff.sh | 27 --- .../selfdrive/pandad/tests/safety_replay.py | 2 +- 5 files changed, 160 insertions(+), 30 deletions(-) create mode 100755 openpilot/selfdrive/pandad/tests/safety_diff.py delete mode 100755 openpilot/selfdrive/pandad/tests/safety_diff.sh diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 73ae7ad004c209..73203aab05078e 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -199,7 +199,7 @@ jobs: restore-keys: safety-replay- - name: Compare safety with the previous opendbc timeout-minutes: 20 - run: openpilot/selfdrive/pandad/tests/safety_diff.sh HEAD^ + run: openpilot/selfdrive/pandad/tests/safety_diff.py simulator_driving: name: simulator driving diff --git a/opendbc_repo b/opendbc_repo index 533372f1299a8f..ec18ed089a32e4 160000 --- a/opendbc_repo +++ b/opendbc_repo @@ -1 +1 @@ -Subproject commit 533372f1299a8f6c9da98623152e2e862197d3fc +Subproject commit ec18ed089a32e4bdc7e259e1f2d0725baf73a48a diff --git a/openpilot/selfdrive/pandad/tests/safety_diff.py b/openpilot/selfdrive/pandad/tests/safety_diff.py new file mode 100755 index 00000000000000..c89d8097a50df7 --- /dev/null +++ b/openpilot/selfdrive/pandad/tests/safety_diff.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python3 +# compares every safety RX and TX result and the safety state on the car test routes, between BASE's opendbc and this one +# safety_diff.py [BASE] BASE defaults to HEAD^, which is master on a PR's merge commit +import hashlib +import json +import multiprocessing +import os +import pickle +import shutil +import subprocess +import sys +import tempfile +from concurrent.futures import ProcessPoolExecutor +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[4] +DT = 0.01 +BUSES = 3 +# a TX message's bus, like cereal's can src +TX = 128 +STATE = ("get_controls_allowed", "get_relay_malfunction", "get_gas_pressed_prev", "get_brake_pressed_prev", "get_regen_braking_prev", + "get_steering_disengage_prev", "get_cruise_engaged_prev", "get_vehicle_moving", "get_acc_main_on", "get_torque_meas_min", + "get_torque_meas_max", "get_torque_driver_min", "get_torque_driver_max", "get_angle_meas_min", "get_angle_meas_max", + "get_curvature_meas_min", "get_curvature_meas_max") + + +def load(route, seconds: float | None = None, tx: bool = False): + from opendbc.car.car_helpers import interfaces + from opendbc.car.tests.test_models import TestCarModelBase + + R = type("R", (TestCarModelBase,), {"test_route": route, "platform": route.car_model}) + car_fw, can_msgs, alpha_long = R.get_testing_data(route) + CP = interfaces[R.platform].get_params(R.platform, R.fingerprint, car_fw, alpha_long, False, docs=False) + cfg = CP.safetyConfigs[-1] + + t0 = can_msgs[0][0] + steps = int((can_msgs[-1][0] - t0) / 1e9 / DT) + 1 if seconds is None else int(seconds / DT) + frames = [[] for _ in range(steps)] + for t, msgs in can_msgs: + i = int((t - t0) / 1e9 / DT) + if i >= len(frames): + break + for m in msgs: + if m.src < 64 and m.src % 4 < BUSES: + frames[i].append((m.address, bytes(m.dat), m.src % 4)) + elif tx and TX <= m.src < TX + 64 and m.src % 4 < BUSES: + frames[i].append((m.address, bytes(m.dat), TX + m.src % 4)) + return cfg.safetyModel.raw, cfg.safetyParam, frames + + +def replay(mode: int, param: int, frames, on_tick=None): + # the result of each message and the safety state after each 10 ms step + from opendbc.safety.tests.libsafety import libsafety_py + safety = libsafety_py.libsafety + assert safety.set_safety_hooks(mode, param) == 0 + safety.set_alternative_experience(0) + + steps = [] + for i, frame in enumerate(frames): + safety.set_timer(int(i * DT * 1e6)) + if i % int(1 / DT) == 0: + safety.safety_tick() + if on_tick is not None: + on_tick(i, safety) + results = [] + for addr, dat, bus in frame: + if bus >= TX: + results.append(bool(safety.safety_tx_hook(libsafety_py.make_CANPacket(addr, bus - TX, dat)))) + else: + fwd = safety.safety_fwd_hook(bus, addr) + results.append((bool(safety.safety_rx_hook(libsafety_py.make_CANPacket(addr, bus, dat))), fwd)) + steps.append((results, tuple(getattr(safety, g)() for g in STATE))) + return steps + + +def dump_route(args): + i, route, out = args + try: + with open(out / f"{i}.pkl", "wb") as f: + pickle.dump((route.route, route.car_model, *load(route, tx=True)), f) + except Exception as e: + return f"{route.route} can't load: {e!r}" + return None + + +def digest_route(path): + # per second: a hash of everything, then what to show if it differs: valid RX, allowed TX and each state value, summed + with open(path, "rb") as f: + route, model, mode, param, frames = pickle.load(f) + steps = replay(mode, param, frames) + seconds = [] + for s in range(0, len(steps), int(1 / DT)): + chunk = steps[s:s + int(1 / DT)] + results = [r for step_results, _ in chunk for r in step_results] + sums = [sum(r[0] for r in results if isinstance(r, tuple)), sum(r for r in results if not isinstance(r, tuple)), + *(int(sum(v)) for v in zip(*(state for _, state in chunk), strict=True))] + seconds.append((hashlib.sha256(repr(chunk).encode()).hexdigest(), sums)) + return route, {"model": model, "seconds": seconds} + + +def pool(): + return ProcessPoolExecutor(os.cpu_count(), mp_context=multiprocessing.get_context("spawn")) + + +def compare(base: dict, new: dict) -> int: + fields = ["valid RX", "allowed TX", *(g.removeprefix("get_") for g in STATE)] + diff = [] + for route in sorted(base): + a, b = base[route], new[route] + changed = [s for s, (x, y) in enumerate(zip(a["seconds"], b["seconds"], strict=True)) if x[0] != y[0]] + if changed: + what = sorted({fields[k] for s in changed for k, (x, y) in enumerate(zip(a["seconds"][s][1], b["seconds"][s][1], strict=True)) if x != y}) + diff.append(route) + print(f"{route} {a['model']}: differs from {changed[0]} s on, in {len(changed)} of {len(a['seconds'])} s: {', '.join(what) or 'order only'}") + print(f"{len(base)} routes, {sum(len(r['seconds']) for r in base.values())} s: " + (f"{len(diff)} routes differ" if diff else "no difference")) + return int(bool(diff)) + + +def main() -> int: + # runs with the base's opendbc, so it only uses libsafety + if sys.argv[1:2] == ["digest"]: + with pool() as p: + digests = dict(p.map(digest_route, sorted(Path(sys.argv[2]).glob("*.pkl")))) + Path(sys.argv[3]).write_text(json.dumps(digests)) + return 0 + + def git(*args, cwd=ROOT): + return subprocess.check_output(["git", *args], cwd=cwd, text=True).strip() + + base_ref = sys.argv[1] if len(sys.argv) > 1 else "HEAD^" + base, new = git("rev-parse", f"{base_ref}:opendbc_repo"), git("rev-parse", "HEAD:opendbc_repo") + if base == new: + print(f"opendbc is unchanged ({new}), nothing to compare") + return 0 + print(f"comparing opendbc {base} -> {new}", flush=True) + + from opendbc.car.tests.routes import routes + with tempfile.TemporaryDirectory() as tmp: + w, odbc = Path(tmp), ROOT / "opendbc_repo" + git("fetch", "-q", "--depth=1", "origin", base, cwd=odbc) + git("worktree", "add", "-q", "--detach", str(w / "base"), base, cwd=odbc) + try: + # the base's safety code, built like this opendbc builds libsafety + shutil.copy(odbc / "opendbc/safety/tests/libsafety/libsafety_py.py", w / "base/opendbc/safety/tests/libsafety/") + (w / "routes").mkdir() + with pool() as p: + for e in filter(None, p.map(dump_route, [(i, r, w / "routes") for i, r in enumerate(routes)])): + print(e) + for name, path in (("base", w / "base"), ("new", odbc)): + subprocess.check_call([sys.executable, __file__, "digest", w / "routes", w / f"{name}.json"], env={**os.environ, "PYTHONPATH": str(path)}) + return compare(json.loads((w / "base.json").read_text()), json.loads((w / "new.json").read_text())) + finally: + git("worktree", "remove", "--force", str(w / "base"), cwd=odbc) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/openpilot/selfdrive/pandad/tests/safety_diff.sh b/openpilot/selfdrive/pandad/tests/safety_diff.sh deleted file mode 100755 index d50151a3fb0518..00000000000000 --- a/openpilot/selfdrive/pandad/tests/safety_diff.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env bash -# compares every safety RX and TX result and the safety state on the car test routes, between BASE's opendbc and this one -set -e - -cd "$(dirname "$0")/../../../.." -BASE=${1:-HEAD^} - -base=$(git rev-parse "$BASE:opendbc_repo") -new=$(git rev-parse HEAD:opendbc_repo) -if [ "$base" = "$new" ]; then - echo "opendbc is unchanged ($new), nothing to compare" - exit 0 -fi -echo "comparing opendbc $base -> $new" - -W=$(mktemp -d) -trap 'git -C opendbc_repo worktree remove --force "$W/base" 2>/dev/null; rm -rf "$W"' EXIT -git -C opendbc_repo fetch -q --depth=1 origin "$base" -git -C opendbc_repo worktree add -q --detach "$W/base" "$base" -# the base's safety code, built like this opendbc builds libsafety -cp opendbc_repo/opendbc/safety/tests/libsafety/libsafety_py.py "$W/base/opendbc/safety/tests/libsafety/" - -REPLAY=opendbc_repo/opendbc/safety/tests/safety_replay/route_replay.py -python3 "$REPLAY" dump "$W/routes" -PYTHONPATH="$W/base" python3 "$REPLAY" digest "$W/routes" "$W/base.json" -python3 "$REPLAY" digest "$W/routes" "$W/new.json" -python3 "$REPLAY" compare "$W/base.json" "$W/new.json" diff --git a/openpilot/selfdrive/pandad/tests/safety_replay.py b/openpilot/selfdrive/pandad/tests/safety_replay.py index 686eb14c968e55..39eaf7f08bc09b 100755 --- a/openpilot/selfdrive/pandad/tests/safety_replay.py +++ b/openpilot/selfdrive/pandad/tests/safety_replay.py @@ -14,10 +14,10 @@ from opendbc.car.tests.routes import routes from opendbc.car.tests.test_models import NUM_JOBS, JOB_ID from opendbc.safety.tests.libsafety import libsafety_py -from opendbc.safety.tests.safety_replay.route_replay import BUSES, DT, load, replay from openpilot.common.basedir import BASEDIR from openpilot.common.hardware import HARDWARE from openpilot.common.utils import retry +from openpilot.selfdrive.pandad.tests.safety_diff import BUSES, DT, load, replay SECONDS_PER_ROUTE = 10 # seconds until the RX checks are compared From 0b889ff91c1bebe129c5c59536a41ef18bd71b2e Mon Sep 17 00:00:00 2001 From: elkoled Date: Mon, 16 Nov 2026 18:53:40 -0800 Subject: [PATCH 12/21] safety replay: review fixes, handle new cars and older opendbc --- .../selfdrive/pandad/tests/safety_diff.py | 35 +++++++++++++------ .../selfdrive/pandad/tests/safety_replay.py | 16 +++++---- 2 files changed, 34 insertions(+), 17 deletions(-) diff --git a/openpilot/selfdrive/pandad/tests/safety_diff.py b/openpilot/selfdrive/pandad/tests/safety_diff.py index c89d8097a50df7..b0795aadac6b95 100755 --- a/openpilot/selfdrive/pandad/tests/safety_diff.py +++ b/openpilot/selfdrive/pandad/tests/safety_diff.py @@ -6,7 +6,6 @@ import multiprocessing import os import pickle -import shutil import subprocess import sys import tempfile @@ -21,7 +20,8 @@ STATE = ("get_controls_allowed", "get_relay_malfunction", "get_gas_pressed_prev", "get_brake_pressed_prev", "get_regen_braking_prev", "get_steering_disengage_prev", "get_cruise_engaged_prev", "get_vehicle_moving", "get_acc_main_on", "get_torque_meas_min", "get_torque_meas_max", "get_torque_driver_min", "get_torque_driver_max", "get_angle_meas_min", "get_angle_meas_max", - "get_curvature_meas_min", "get_curvature_meas_max") + "get_curvature_meas_min", "get_curvature_meas_max", "get_vehicle_speed_min", "get_vehicle_speed_max", "get_desired_curvature_last", + "get_honda_fwd_brake") def load(route, seconds: float | None = None, tx: bool = False): @@ -43,7 +43,8 @@ def load(route, seconds: float | None = None, tx: bool = False): for m in msgs: if m.src < 64 and m.src % 4 < BUSES: frames[i].append((m.address, bytes(m.dat), m.src % 4)) - elif tx and TX <= m.src < TX + 64 and m.src % 4 < BUSES: + # sent and blocked TX, so a check that allows more shows up too + elif tx and m.src >= TX and m.src % 4 < BUSES: frames[i].append((m.address, bytes(m.dat), TX + m.src % 4)) return cfg.safetyModel.raw, cfg.safetyParam, frames @@ -54,12 +55,14 @@ def replay(mode: int, param: int, frames, on_tick=None): safety = libsafety_py.libsafety assert safety.set_safety_hooks(mode, param) == 0 safety.set_alternative_experience(0) + # older opendbc names it safety_tick_current_safety_config + tick = safety.safety_tick if hasattr(safety, "safety_tick") else safety.safety_tick_current_safety_config steps = [] for i, frame in enumerate(frames): safety.set_timer(int(i * DT * 1e6)) if i % int(1 / DT) == 0: - safety.safety_tick() + tick() if on_tick is not None: on_tick(i, safety) results = [] @@ -77,7 +80,7 @@ def dump_route(args): i, route, out = args try: with open(out / f"{i}.pkl", "wb") as f: - pickle.dump((route.route, route.car_model, *load(route, tx=True)), f) + pickle.dump((route.route, str(route.car_model), *load(route, tx=True)), f) except Exception as e: return f"{route.route} can't load: {e!r}" return None @@ -87,13 +90,16 @@ def digest_route(path): # per second: a hash of everything, then what to show if it differs: valid RX, allowed TX and each state value, summed with open(path, "rb") as f: route, model, mode, param, frames = pickle.load(f) + from opendbc.safety.tests.libsafety import libsafety_py + if libsafety_py.libsafety.set_safety_hooks(mode, param) != 0: + return route, {"model": model, "seconds": None} steps = replay(mode, param, frames) seconds = [] for s in range(0, len(steps), int(1 / DT)): chunk = steps[s:s + int(1 / DT)] results = [r for step_results, _ in chunk for r in step_results] sums = [sum(r[0] for r in results if isinstance(r, tuple)), sum(r for r in results if not isinstance(r, tuple)), - *(int(sum(v)) for v in zip(*(state for _, state in chunk), strict=True))] + *(sum(v) for v in zip(*(state for _, state in chunk), strict=True))] seconds.append((hashlib.sha256(repr(chunk).encode()).hexdigest(), sums)) return route, {"model": model, "seconds": seconds} @@ -107,12 +113,19 @@ def compare(base: dict, new: dict) -> int: diff = [] for route in sorted(base): a, b = base[route], new[route] + if b["seconds"] is None: + diff.append(route) + print(f"{route} {a['model']}: this opendbc doesn't support its safety mode") + continue + if a["seconds"] is None: + print(f"{route} {a['model']}: new, the base opendbc doesn't support its safety mode") + continue changed = [s for s, (x, y) in enumerate(zip(a["seconds"], b["seconds"], strict=True)) if x[0] != y[0]] if changed: what = sorted({fields[k] for s in changed for k, (x, y) in enumerate(zip(a["seconds"][s][1], b["seconds"][s][1], strict=True)) if x != y}) diff.append(route) print(f"{route} {a['model']}: differs from {changed[0]} s on, in {len(changed)} of {len(a['seconds'])} s: {', '.join(what) or 'order only'}") - print(f"{len(base)} routes, {sum(len(r['seconds']) for r in base.values())} s: " + (f"{len(diff)} routes differ" if diff else "no difference")) + print(f"{len(base)} routes, {sum(len(r['seconds'] or []) for r in new.values())} s: " + (f"{len(diff)} routes differ" if diff else "no difference")) return int(bool(diff)) @@ -140,12 +153,12 @@ def git(*args, cwd=ROOT): git("fetch", "-q", "--depth=1", "origin", base, cwd=odbc) git("worktree", "add", "-q", "--detach", str(w / "base"), base, cwd=odbc) try: - # the base's safety code, built like this opendbc builds libsafety - shutil.copy(odbc / "opendbc/safety/tests/libsafety/libsafety_py.py", w / "base/opendbc/safety/tests/libsafety/") (w / "routes").mkdir() with pool() as p: - for e in filter(None, p.map(dump_route, [(i, r, w / "routes") for i, r in enumerate(routes)])): - print(e) + errors = [e for e in p.map(dump_route, [(i, r, w / "routes") for i, r in enumerate(routes)]) if e] + if errors: + print("\n".join(errors)) + return 1 for name, path in (("base", w / "base"), ("new", odbc)): subprocess.check_call([sys.executable, __file__, "digest", w / "routes", w / f"{name}.json"], env={**os.environ, "PYTHONPATH": str(path)}) return compare(json.loads((w / "base.json").read_text()), json.loads((w / "new.json").read_text())) diff --git a/openpilot/selfdrive/pandad/tests/safety_replay.py b/openpilot/selfdrive/pandad/tests/safety_replay.py index 39eaf7f08bc09b..82dda77b6c2e9b 100755 --- a/openpilot/selfdrive/pandad/tests/safety_replay.py +++ b/openpilot/selfdrive/pandad/tests/safety_replay.py @@ -16,7 +16,6 @@ from opendbc.safety.tests.libsafety import libsafety_py from openpilot.common.basedir import BASEDIR from openpilot.common.hardware import HARDWARE -from openpilot.common.utils import retry from openpilot.selfdrive.pandad.tests.safety_diff import BUSES, DT, load, replay SECONDS_PER_ROUTE = 10 @@ -35,7 +34,7 @@ def inputs() -> str: files = sorted((root / "panda/board/obj").glob("*.bin.signed")) for d in ("opendbc_repo/opendbc", "panda/python"): files += sorted(f for f in (root / d).rglob("*") if f.is_file() and f.suffix not in BUILD_OUTPUTS and "__pycache__" not in f.parts) - files.append(Path(__file__).resolve()) + files += [Path(__file__).resolve(), Path(__file__).with_name("safety_diff.py").resolve(), root / "openpilot/selfdrive/pandad/pandad.py"] h = hashlib.sha256() for f in files: h.update(str(f.relative_to(root)).encode() + b"\0" + f.read_bytes()) @@ -50,13 +49,18 @@ def passed_age(key: str) -> float | None: age = time.time() - float((PASSED / key).read_text()) # noqa: TID251 except (OSError, ValueError): return None - return age if age < PASS_TTL else None + return age if 0 <= age < PASS_TTL else None -# a route download can drop mid file -@retry(attempts=3, delay=10) def load_window(route): - return load(route, SECONDS_PER_ROUTE) + # a route download can drop mid file + for attempt in range(3): + try: + return load(route, SECONDS_PER_ROUTE) + except Exception: + if attempt == 2: + raise + time.sleep(10) def without_forwarded_copies(mode, param, frames): From 4f8a7e7221689433e6409c48885ff36a9ac01059 Mon Sep 17 00:00:00 2001 From: elkoled Date: Mon, 16 Nov 2026 18:58:00 -0800 Subject: [PATCH 13/21] safety replay: allow for the UP's CAN lanes starting a few frames apart --- .../selfdrive/pandad/tests/safety_replay.py | 24 +++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/openpilot/selfdrive/pandad/tests/safety_replay.py b/openpilot/selfdrive/pandad/tests/safety_replay.py index 82dda77b6c2e9b..f5ceb15c2e2ac2 100755 --- a/openpilot/selfdrive/pandad/tests/safety_replay.py +++ b/openpilot/selfdrive/pandad/tests/safety_replay.py @@ -21,6 +21,8 @@ SECONDS_PER_ROUTE = 10 # seconds until the RX checks are compared STEADY = 2 +# frames the station's CAN lanes can start apart +SKEW = 3 # master's passing runs, restored with the alfred cache PASSED = Path("/data/scons_cache/safety_replay") # a pass is reused for a day, so the rack itself is still checked daily @@ -104,11 +106,29 @@ def on_tick(i, safety): "safety": [mode, param], "faults": 0, "tx_buffer_overflow": 0, "heartbeat_disabled": False, "harness_changed": False} +def skewed(frames, bus, k): + # this bus's lane starts k frames later + out = [[m for m in f if m[2] != bus] for f in [[]] * SKEW + frames + [[]] * SKEW] + for i, f in enumerate(frames): + out[SKEW + i + k] += [m for m in f if m[2] == bus] + return out + + +def bounds(runs): + def pick(fn, vals): + if isinstance(vals[0], list): + return [pick(fn, list(v)) for v in zip(*vals, strict=True)] + return fn(vals) + return ({k: pick(min, [r[k] for r in runs]) for k in runs[0]}, {k: pick(max, [r[k] for r in runs]) for k in runs[0]}) + + def prepare(route): mode, param, frames = load_window(route) frames = without_forwarded_copies(mode, param, frames) - # the UP sends a 10 ms step's frames on different buses in any order - lo, hi = (expected(mode, param, [sorted(f, key=lambda m: m[2] * order) for f in frames]) for order in (1, -1)) + # the UP sends a 10 ms step's frames on different buses in any order, and its lanes start up to SKEW frames apart + runs = [expected(mode, param, [sorted(f, key=lambda m: m[2] * order) for f in skewed(frames, 2, k)]) + for order in (1, -1) for k in range(-SKEW, SKEW + 1)] + lo, hi = bounds(runs) return mode, param, frames, lo, hi From f93f4be584340189df1ff02d7a2cb1eff38f8f42 Mon Sep 17 00:00:00 2001 From: elkoled Date: Tue, 17 Nov 2026 06:41:35 -0800 Subject: [PATCH 14/21] safety replay: diff_safety.py reports every change in the diff report comment --- .github/workflows/diff_report.yaml | 29 ++- .github/workflows/tests.yaml | 9 +- .../selfdrive/pandad/tests/diff_safety.py | 238 ++++++++++++++++++ .../selfdrive/pandad/tests/safety_diff.py | 170 ------------- .../selfdrive/pandad/tests/safety_replay.py | 4 +- 5 files changed, 275 insertions(+), 175 deletions(-) create mode 100755 openpilot/selfdrive/pandad/tests/diff_safety.py delete mode 100755 openpilot/selfdrive/pandad/tests/safety_diff.py diff --git a/.github/workflows/diff_report.yaml b/.github/workflows/diff_report.yaml index 202bc0ec7934af..471efc2dfcc260 100644 --- a/.github/workflows/diff_report.yaml +++ b/.github/workflows/diff_report.yaml @@ -48,11 +48,36 @@ jobs: run-id: ${{ steps.tests-run.outputs.run-id }} name: diff_report_${{ github.event.number }} path: . + - name: Wait for safety replay + if: steps.wait.outcome == 'success' + id: wait-safety + continue-on-error: true + uses: lewagon/wait-on-check-action@1d57e2c51a58d812d2765e036a028b6bdb5a6154 + with: + ref: ${{ github.event.pull_request.head.sha }} + check-name: safety replay + repo-token: ${{ secrets.GITHUB_TOKEN }} + allowed-conclusions: success,failure + wait-interval: 20 + - name: Download safety diff + if: steps.wait-safety.outcome == 'success' + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + repository: ${{ github.repository }} + run-id: ${{ steps.tests-run.outputs.run-id }} + name: diff_safety_${{ github.event.number }} + path: . + # one comment for both reports, the safety one only when this PR changes opendbc + - name: Combine reports + if: steps.wait.outcome == 'success' + run: cat diff_report.txt diff_safety.txt > comment.md 2>/dev/null; [ -s comment.md ] || rm -f comment.md - name: Comment on PR - if: steps.wait.outcome == 'success' && hashFiles('diff_report.txt') != '' + if: steps.wait.outcome == 'success' && hashFiles('comment.md') != '' uses: thollander/actions-comment-pull-request@24bffb9b452ba05a4f3f77933840a6a841d1b32b with: - file-path: diff_report.txt + file-path: comment.md comment-tag: diff_report pr-number: ${{ github.event.number }} github-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 73203aab05078e..fc456889f6c27d 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -199,7 +199,14 @@ jobs: restore-keys: safety-replay- - name: Compare safety with the previous opendbc timeout-minutes: 20 - run: openpilot/selfdrive/pandad/tests/safety_diff.py + run: openpilot/selfdrive/pandad/tests/diff_safety.py + - name: Upload diff report + if: always() && github.event_name == 'pull_request' + uses: actions/upload-artifact@v7 + with: + name: diff_safety_${{ github.event.number }} + path: diff_safety.txt + if-no-files-found: ignore simulator_driving: name: simulator driving diff --git a/openpilot/selfdrive/pandad/tests/diff_safety.py b/openpilot/selfdrive/pandad/tests/diff_safety.py new file mode 100755 index 00000000000000..9f68bba88031c7 --- /dev/null +++ b/openpilot/selfdrive/pandad/tests/diff_safety.py @@ -0,0 +1,238 @@ +#!/usr/bin/env python3 +# replays the car test routes through BASE's and this opendbc's safety, reports every changed RX and TX result and safety state value +# diff_safety.py [BASE] BASE defaults to HEAD^, which is master on a PR's merge commit +import multiprocessing +import os +import pickle +import subprocess +import sys +import tempfile +from collections import Counter +from concurrent.futures import ProcessPoolExecutor +from pathlib import Path + +import numpy as np + +ROOT = Path(__file__).resolve().parents[4] +REPORT = ROOT / "diff_safety.txt" +DT = 0.01 +BUSES = 3 +# a TX message's bus, like cereal's can src +TX = 128 +STATE = ("get_controls_allowed", "get_relay_malfunction", "get_gas_pressed_prev", "get_brake_pressed_prev", "get_regen_braking_prev", + "get_steering_disengage_prev", "get_cruise_engaged_prev", "get_vehicle_moving", "get_acc_main_on", "get_torque_meas_min", + "get_torque_meas_max", "get_torque_driver_min", "get_torque_driver_max", "get_angle_meas_min", "get_angle_meas_max", + "get_curvature_meas_min", "get_curvature_meas_max", "get_vehicle_speed_min", "get_vehicle_speed_max", "get_desired_curvature_last", + "get_honda_fwd_brake") +BOOL_STATE = {"controls_allowed", "relay_malfunction", "gas_pressed_prev", "brake_pressed_prev", "regen_braking_prev", "steering_disengage_prev", + "cruise_engaged_prev", "vehicle_moving", "acc_main_on", "honda_fwd_brake"} +MAX_GROUPS = 3 + + +def load(route, seconds: float | None = None, tx: bool = False): + from opendbc.car.car_helpers import interfaces + from opendbc.car.tests.test_models import TestCarModelBase + + R = type("R", (TestCarModelBase,), {"test_route": route, "platform": route.car_model}) + car_fw, can_msgs, alpha_long = R.get_testing_data(route) + CP = interfaces[R.platform].get_params(R.platform, R.fingerprint, car_fw, alpha_long, False, docs=False) + cfg = CP.safetyConfigs[-1] + + t0 = can_msgs[0][0] + steps = int((can_msgs[-1][0] - t0) / 1e9 / DT) + 1 if seconds is None else int(seconds / DT) + frames = [[] for _ in range(steps)] + for t, msgs in can_msgs: + i = int((t - t0) / 1e9 / DT) + if i >= len(frames): + break + for m in msgs: + if m.src < 64 and m.src % 4 < BUSES: + frames[i].append((m.address, bytes(m.dat), m.src % 4)) + # sent and blocked TX, so a check that allows more shows up too + elif tx and m.src >= TX and m.src % 4 < BUSES: + frames[i].append((m.address, bytes(m.dat), TX + m.src % 4)) + return cfg.safetyModel.raw, cfg.safetyParam, frames + + +def replay(mode: int, param: int, frames, on_tick=None): + # the result of each message and the safety state after each 10 ms step + from opendbc.safety.tests.libsafety import libsafety_py + safety = libsafety_py.libsafety + assert safety.set_safety_hooks(mode, param) == 0 + safety.set_alternative_experience(0) + # older opendbc names it safety_tick_current_safety_config + tick = safety.safety_tick if hasattr(safety, "safety_tick") else safety.safety_tick_current_safety_config + + steps = [] + for i, frame in enumerate(frames): + safety.set_timer(int(i * DT * 1e6)) + if i % int(1 / DT) == 0: + tick() + if on_tick is not None: + on_tick(i, safety) + results = [] + for addr, dat, bus in frame: + if bus >= TX: + results.append(bool(safety.safety_tx_hook(libsafety_py.make_CANPacket(addr, bus - TX, dat)))) + else: + fwd = safety.safety_fwd_hook(bus, addr) + results.append((bool(safety.safety_rx_hook(libsafety_py.make_CANPacket(addr, bus, dat))), fwd)) + steps.append((results, tuple(getattr(safety, g)() for g in STATE))) + return steps + + +def dump_route(args): + i, route, out = args + try: + with open(out / f"{i}.pkl", "wb") as f: + pickle.dump((route.route, str(route.car_model), *load(route, tx=True)), f) + except Exception as e: + return f"ERROR {route.car_model} - {route.route}: can't load, {e!r}" + return None + + +def results_route(args): + # every message's result (RX valid or TX allowed), where each RX is forwarded, and the state after each step + path, out = args + with open(path, "rb") as f: + route, model, mode, param, frames = pickle.load(f) + from opendbc.safety.tests.libsafety import libsafety_py + if libsafety_py.libsafety.set_safety_hooks(mode, param) != 0: + np.savez(out / path.with_suffix(".npz").name, mode=mode) + return + steps = replay(mode, param, frames) + results = [r for step_results, _ in steps for r in step_results] + np.savez(out / path.with_suffix(".npz").name, + ok=np.array([r[0] if isinstance(r, tuple) else r for r in results], dtype=bool), + fwd=np.array([r[1] if isinstance(r, tuple) else -2 for r in results], dtype=np.int16), + state=np.array([state for _, state in steps], dtype=np.float64)) + + +def pool(): + return ProcessPoolExecutor(os.cpu_count(), mp_context=multiprocessing.get_context("spawn")) + + +def value(v: float, name: str): + if name in BOOL_STATE: + return bool(v) + return int(v) if v == int(v) else round(v, 3) + + +def message_diffs(frames, a, b) -> list[str]: + # RX valid, RX forward and TX allowed, grouped by message and change + events = [(i, addr, bus) for i, frame in enumerate(frames) for addr, _, bus in frame] + changes = Counter() + first = {} + for k in np.flatnonzero((a["ok"] != b["ok"]) | (a["fwd"] != b["fwd"])): + i, addr, bus = events[k] + if bus >= TX: + key = (f"TX 0x{addr:x} bus {bus - TX} allowed", bool(a["ok"][k]), bool(b["ok"][k])) + elif a["ok"][k] != b["ok"][k]: + key = (f"RX 0x{addr:x} bus {bus} valid", bool(a["ok"][k]), bool(b["ok"][k])) + else: + key = (f"RX 0x{addr:x} bus {bus} forward", int(a["fwd"][k]), int(b["fwd"][k])) + changes[key] += 1 + first.setdefault(key, i) + return [f"\n {what}: {old} -> {new} ({n} msgs, from frame {first[(what, old, new)]})" for (what, old, new), n in sorted(changes.items())] + + +def state_diffs(a, b) -> list[str]: + from opendbc.car.tests.car_diff import PADDING, find_edges, format_numeric_diffs, format_timing, group_frames, render_waveform + lines = [] + for k, getter in enumerate(STATE): + name = getter.removeprefix("get_") + old, new = a["state"][:, k], b["state"][:, k] + frames = np.flatnonzero(old != new) + if not len(frames): + continue + diffs = [(name, int(i), (value(old[i], name), value(new[i], name)), int(i * DT * 1e9)) for i in frames] + lines.append(f"\n {name} ({len(diffs)} diffs)") + if name not in BOOL_STATE: + lines += format_numeric_diffs(diffs) + continue + groups = group_frames(diffs) + for group in groups[:MAX_GROUPS]: + start, end = max(0, group[0][1] - PADDING), min(group[-1][1] + PADDING + 1, len(old)) + master, pr = [bool(v) for v in old[start:end]], [bool(v) for v in new[start:end]] + lines += [f"\n frames {start}-{end - 1}", render_waveform("master", master), render_waveform("PR", pr)] + for edge, m, p in zip(("rise", "fall"), find_edges(master), find_edges(pr), strict=True): + timing = format_timing(edge, m, p, DT * 1000) + if timing: + lines.append(timing) + if len(groups) > MAX_GROUPS: + lines.append(f" (... {len(groups) - MAX_GROUPS} more)") + return lines + + +def diff_report(dumps: Path, base: Path, new: Path, errors: list[str]) -> int: + changed, new_cars, n_passed = [], [], 0 + for dump in sorted(dumps.glob("*.pkl"), key=lambda p: int(p.stem)): + with open(dump, "rb") as f: + route, model, mode, _, frames = pickle.load(f) + a, b = np.load(base / f"{dump.stem}.npz"), np.load(new / f"{dump.stem}.npz") + if "ok" not in b: + errors.append(f"ERROR {model} - {route}: this opendbc has no safety mode {mode}") + elif "ok" not in a: + new_cars.append(f"\nNEW {model} - {route}: master's opendbc has no safety mode {mode}") + else: + lines = message_diffs(frames, a, b) + state_diffs(a, b) + if lines: + changed.append([f"\n{model} - {route}"] + lines) + else: + n_passed += 1 + + icon = "⚠️" if changed or errors else "✅" + lines = [ + "## Safety replay diff report", + "Replays every car test route through master's and this PR's panda safety and compares each RX and TX result and the safety state.", + "Please review any changes carefully to ensure they are expected.\n", + f"{icon} {len(changed)} changed, {n_passed} passed, {len(errors)} errors" + (f", {len(new_cars)} new" if new_cars else ""), + ] + lines += [f"\n{e}" for e in errors] + new_cars + if changed: + lines.append("
Show changes\n\n```") + for route_lines in changed: + lines += route_lines + lines.append("```\n
") + REPORT.write_text("\n".join(lines) + "\n") + print("\n".join(lines)) + return int(bool(changed or errors)) + + +def main() -> int: + # runs with the base's opendbc, so it only uses libsafety + if sys.argv[1:2] == ["results"]: + dumps, out = Path(sys.argv[2]), Path(sys.argv[3]) + out.mkdir() + with pool() as p: + list(p.map(results_route, [(d, out) for d in sorted(dumps.glob("*.pkl"))])) + return 0 + + def git(*args, cwd=ROOT): + return subprocess.check_output(["git", *args], cwd=cwd, text=True).strip() + + base_ref = sys.argv[1] if len(sys.argv) > 1 else "HEAD^" + base, new = git("rev-parse", f"{base_ref}:opendbc_repo"), git("rev-parse", "HEAD:opendbc_repo") + if base == new: + print(f"opendbc is unchanged ({new}), nothing to compare") + return 0 + print(f"comparing opendbc {base} -> {new}", flush=True) + + from opendbc.car.tests.routes import routes + with tempfile.TemporaryDirectory() as tmp: + w, odbc = Path(tmp), ROOT / "opendbc_repo" + git("fetch", "-q", "--depth=1", "origin", base, cwd=odbc) + git("worktree", "add", "-q", "--detach", str(w / "base"), base, cwd=odbc) + try: + (w / "routes").mkdir() + with pool() as p: + errors = [e for e in p.map(dump_route, [(i, r, w / "routes") for i, r in enumerate(routes)]) if e] + for name, path in (("base", w / "base"), ("new", odbc)): + subprocess.check_call([sys.executable, __file__, "results", w / "routes", w / f"{name}_results"], env={**os.environ, "PYTHONPATH": str(path)}) + return diff_report(w / "routes", w / "base_results", w / "new_results", errors) + finally: + git("worktree", "remove", "--force", str(w / "base"), cwd=odbc) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/openpilot/selfdrive/pandad/tests/safety_diff.py b/openpilot/selfdrive/pandad/tests/safety_diff.py deleted file mode 100755 index b0795aadac6b95..00000000000000 --- a/openpilot/selfdrive/pandad/tests/safety_diff.py +++ /dev/null @@ -1,170 +0,0 @@ -#!/usr/bin/env python3 -# compares every safety RX and TX result and the safety state on the car test routes, between BASE's opendbc and this one -# safety_diff.py [BASE] BASE defaults to HEAD^, which is master on a PR's merge commit -import hashlib -import json -import multiprocessing -import os -import pickle -import subprocess -import sys -import tempfile -from concurrent.futures import ProcessPoolExecutor -from pathlib import Path - -ROOT = Path(__file__).resolve().parents[4] -DT = 0.01 -BUSES = 3 -# a TX message's bus, like cereal's can src -TX = 128 -STATE = ("get_controls_allowed", "get_relay_malfunction", "get_gas_pressed_prev", "get_brake_pressed_prev", "get_regen_braking_prev", - "get_steering_disengage_prev", "get_cruise_engaged_prev", "get_vehicle_moving", "get_acc_main_on", "get_torque_meas_min", - "get_torque_meas_max", "get_torque_driver_min", "get_torque_driver_max", "get_angle_meas_min", "get_angle_meas_max", - "get_curvature_meas_min", "get_curvature_meas_max", "get_vehicle_speed_min", "get_vehicle_speed_max", "get_desired_curvature_last", - "get_honda_fwd_brake") - - -def load(route, seconds: float | None = None, tx: bool = False): - from opendbc.car.car_helpers import interfaces - from opendbc.car.tests.test_models import TestCarModelBase - - R = type("R", (TestCarModelBase,), {"test_route": route, "platform": route.car_model}) - car_fw, can_msgs, alpha_long = R.get_testing_data(route) - CP = interfaces[R.platform].get_params(R.platform, R.fingerprint, car_fw, alpha_long, False, docs=False) - cfg = CP.safetyConfigs[-1] - - t0 = can_msgs[0][0] - steps = int((can_msgs[-1][0] - t0) / 1e9 / DT) + 1 if seconds is None else int(seconds / DT) - frames = [[] for _ in range(steps)] - for t, msgs in can_msgs: - i = int((t - t0) / 1e9 / DT) - if i >= len(frames): - break - for m in msgs: - if m.src < 64 and m.src % 4 < BUSES: - frames[i].append((m.address, bytes(m.dat), m.src % 4)) - # sent and blocked TX, so a check that allows more shows up too - elif tx and m.src >= TX and m.src % 4 < BUSES: - frames[i].append((m.address, bytes(m.dat), TX + m.src % 4)) - return cfg.safetyModel.raw, cfg.safetyParam, frames - - -def replay(mode: int, param: int, frames, on_tick=None): - # the result of each message and the safety state after each 10 ms step - from opendbc.safety.tests.libsafety import libsafety_py - safety = libsafety_py.libsafety - assert safety.set_safety_hooks(mode, param) == 0 - safety.set_alternative_experience(0) - # older opendbc names it safety_tick_current_safety_config - tick = safety.safety_tick if hasattr(safety, "safety_tick") else safety.safety_tick_current_safety_config - - steps = [] - for i, frame in enumerate(frames): - safety.set_timer(int(i * DT * 1e6)) - if i % int(1 / DT) == 0: - tick() - if on_tick is not None: - on_tick(i, safety) - results = [] - for addr, dat, bus in frame: - if bus >= TX: - results.append(bool(safety.safety_tx_hook(libsafety_py.make_CANPacket(addr, bus - TX, dat)))) - else: - fwd = safety.safety_fwd_hook(bus, addr) - results.append((bool(safety.safety_rx_hook(libsafety_py.make_CANPacket(addr, bus, dat))), fwd)) - steps.append((results, tuple(getattr(safety, g)() for g in STATE))) - return steps - - -def dump_route(args): - i, route, out = args - try: - with open(out / f"{i}.pkl", "wb") as f: - pickle.dump((route.route, str(route.car_model), *load(route, tx=True)), f) - except Exception as e: - return f"{route.route} can't load: {e!r}" - return None - - -def digest_route(path): - # per second: a hash of everything, then what to show if it differs: valid RX, allowed TX and each state value, summed - with open(path, "rb") as f: - route, model, mode, param, frames = pickle.load(f) - from opendbc.safety.tests.libsafety import libsafety_py - if libsafety_py.libsafety.set_safety_hooks(mode, param) != 0: - return route, {"model": model, "seconds": None} - steps = replay(mode, param, frames) - seconds = [] - for s in range(0, len(steps), int(1 / DT)): - chunk = steps[s:s + int(1 / DT)] - results = [r for step_results, _ in chunk for r in step_results] - sums = [sum(r[0] for r in results if isinstance(r, tuple)), sum(r for r in results if not isinstance(r, tuple)), - *(sum(v) for v in zip(*(state for _, state in chunk), strict=True))] - seconds.append((hashlib.sha256(repr(chunk).encode()).hexdigest(), sums)) - return route, {"model": model, "seconds": seconds} - - -def pool(): - return ProcessPoolExecutor(os.cpu_count(), mp_context=multiprocessing.get_context("spawn")) - - -def compare(base: dict, new: dict) -> int: - fields = ["valid RX", "allowed TX", *(g.removeprefix("get_") for g in STATE)] - diff = [] - for route in sorted(base): - a, b = base[route], new[route] - if b["seconds"] is None: - diff.append(route) - print(f"{route} {a['model']}: this opendbc doesn't support its safety mode") - continue - if a["seconds"] is None: - print(f"{route} {a['model']}: new, the base opendbc doesn't support its safety mode") - continue - changed = [s for s, (x, y) in enumerate(zip(a["seconds"], b["seconds"], strict=True)) if x[0] != y[0]] - if changed: - what = sorted({fields[k] for s in changed for k, (x, y) in enumerate(zip(a["seconds"][s][1], b["seconds"][s][1], strict=True)) if x != y}) - diff.append(route) - print(f"{route} {a['model']}: differs from {changed[0]} s on, in {len(changed)} of {len(a['seconds'])} s: {', '.join(what) or 'order only'}") - print(f"{len(base)} routes, {sum(len(r['seconds'] or []) for r in new.values())} s: " + (f"{len(diff)} routes differ" if diff else "no difference")) - return int(bool(diff)) - - -def main() -> int: - # runs with the base's opendbc, so it only uses libsafety - if sys.argv[1:2] == ["digest"]: - with pool() as p: - digests = dict(p.map(digest_route, sorted(Path(sys.argv[2]).glob("*.pkl")))) - Path(sys.argv[3]).write_text(json.dumps(digests)) - return 0 - - def git(*args, cwd=ROOT): - return subprocess.check_output(["git", *args], cwd=cwd, text=True).strip() - - base_ref = sys.argv[1] if len(sys.argv) > 1 else "HEAD^" - base, new = git("rev-parse", f"{base_ref}:opendbc_repo"), git("rev-parse", "HEAD:opendbc_repo") - if base == new: - print(f"opendbc is unchanged ({new}), nothing to compare") - return 0 - print(f"comparing opendbc {base} -> {new}", flush=True) - - from opendbc.car.tests.routes import routes - with tempfile.TemporaryDirectory() as tmp: - w, odbc = Path(tmp), ROOT / "opendbc_repo" - git("fetch", "-q", "--depth=1", "origin", base, cwd=odbc) - git("worktree", "add", "-q", "--detach", str(w / "base"), base, cwd=odbc) - try: - (w / "routes").mkdir() - with pool() as p: - errors = [e for e in p.map(dump_route, [(i, r, w / "routes") for i, r in enumerate(routes)]) if e] - if errors: - print("\n".join(errors)) - return 1 - for name, path in (("base", w / "base"), ("new", odbc)): - subprocess.check_call([sys.executable, __file__, "digest", w / "routes", w / f"{name}.json"], env={**os.environ, "PYTHONPATH": str(path)}) - return compare(json.loads((w / "base.json").read_text()), json.loads((w / "new.json").read_text())) - finally: - git("worktree", "remove", "--force", str(w / "base"), cwd=odbc) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/openpilot/selfdrive/pandad/tests/safety_replay.py b/openpilot/selfdrive/pandad/tests/safety_replay.py index f5ceb15c2e2ac2..85546ffab0fd16 100755 --- a/openpilot/selfdrive/pandad/tests/safety_replay.py +++ b/openpilot/selfdrive/pandad/tests/safety_replay.py @@ -16,7 +16,7 @@ from opendbc.safety.tests.libsafety import libsafety_py from openpilot.common.basedir import BASEDIR from openpilot.common.hardware import HARDWARE -from openpilot.selfdrive.pandad.tests.safety_diff import BUSES, DT, load, replay +from openpilot.selfdrive.pandad.tests.diff_safety import BUSES, DT, load, replay SECONDS_PER_ROUTE = 10 # seconds until the RX checks are compared @@ -36,7 +36,7 @@ def inputs() -> str: files = sorted((root / "panda/board/obj").glob("*.bin.signed")) for d in ("opendbc_repo/opendbc", "panda/python"): files += sorted(f for f in (root / d).rglob("*") if f.is_file() and f.suffix not in BUILD_OUTPUTS and "__pycache__" not in f.parts) - files += [Path(__file__).resolve(), Path(__file__).with_name("safety_diff.py").resolve(), root / "openpilot/selfdrive/pandad/pandad.py"] + files += [Path(__file__).resolve(), Path(__file__).with_name("diff_safety.py").resolve(), root / "openpilot/selfdrive/pandad/pandad.py"] h = hashlib.sha256() for f in files: h.update(str(f.relative_to(root)).encode() + b"\0" + f.read_bytes()) From 3af1361f5a65a2b93ea0391dd01e20ee4bf92a20 Mon Sep 17 00:00:00 2001 From: elkoled Date: Tue, 17 Nov 2026 06:58:22 -0800 Subject: [PATCH 15/21] diff_safety: boolean waveforms through car_diff.format_diff --- .../selfdrive/pandad/tests/diff_safety.py | 28 ++++++++----------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/openpilot/selfdrive/pandad/tests/diff_safety.py b/openpilot/selfdrive/pandad/tests/diff_safety.py index 9f68bba88031c7..ddb91f0cd4c20d 100755 --- a/openpilot/selfdrive/pandad/tests/diff_safety.py +++ b/openpilot/selfdrive/pandad/tests/diff_safety.py @@ -26,7 +26,6 @@ "get_honda_fwd_brake") BOOL_STATE = {"controls_allowed", "relay_malfunction", "gas_pressed_prev", "brake_pressed_prev", "regen_braking_prev", "steering_disengage_prev", "cruise_engaged_prev", "vehicle_moving", "acc_main_on", "honda_fwd_brake"} -MAX_GROUPS = 3 def load(route, seconds: float | None = None, tx: bool = False): @@ -136,8 +135,16 @@ def message_diffs(frames, a, b) -> list[str]: return [f"\n {what}: {old} -> {new} ({n} msgs, from frame {first[(what, old, new)]})" for (what, old, new), n in sorted(changes.items())] +class StateWrap: + """Adapter so car_diff reads one safety state value""" + def __init__(self, name: str, v): + self._d = {name: v} + def to_dict(self) -> dict: + return self._d + + def state_diffs(a, b) -> list[str]: - from opendbc.car.tests.car_diff import PADDING, find_edges, format_numeric_diffs, format_timing, group_frames, render_waveform + from opendbc.car.tests.car_diff import format_diff lines = [] for k, getter in enumerate(STATE): name = getter.removeprefix("get_") @@ -146,21 +153,10 @@ def state_diffs(a, b) -> list[str]: if not len(frames): continue diffs = [(name, int(i), (value(old[i], name), value(new[i], name)), int(i * DT * 1e9)) for i in frames] + ref = [(int(i * DT * 1e9), StateWrap(name, value(v, name))) for i, v in enumerate(old)] + states = [StateWrap(name, value(v, name)) for v in new] lines.append(f"\n {name} ({len(diffs)} diffs)") - if name not in BOOL_STATE: - lines += format_numeric_diffs(diffs) - continue - groups = group_frames(diffs) - for group in groups[:MAX_GROUPS]: - start, end = max(0, group[0][1] - PADDING), min(group[-1][1] + PADDING + 1, len(old)) - master, pr = [bool(v) for v in old[start:end]], [bool(v) for v in new[start:end]] - lines += [f"\n frames {start}-{end - 1}", render_waveform("master", master), render_waveform("PR", pr)] - for edge, m, p in zip(("rise", "fall"), find_edges(master), find_edges(pr), strict=True): - timing = format_timing(edge, m, p, DT * 1000) - if timing: - lines.append(timing) - if len(groups) > MAX_GROUPS: - lines.append(f" (... {len(groups) - MAX_GROUPS} more)") + lines += format_diff(diffs, ref, states, name) return lines From aa494e5f1b640362620b7240f35d12e9bb692907 Mon Sep 17 00:00:00 2001 From: elkoled Date: Tue, 17 Nov 2026 07:01:34 -0800 Subject: [PATCH 16/21] diff report: share the report layout with the safety replay --- .github/workflows/tests.yaml | 3 ++ .../selfdrive/pandad/tests/diff_safety.py | 25 +++------ .../test/process_replay/diff_report.py | 54 +++++++++++-------- 3 files changed, 42 insertions(+), 40 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index fc456889f6c27d..1324a5551fed9e 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -197,6 +197,9 @@ jobs: path: /tmp/comma_download_cache key: safety-replay-${{ hashFiles('opendbc_repo/opendbc/car/tests/routes.py') }} restore-keys: safety-replay- + # the report reuses process replay's diff report layout + - name: Build openpilot + run: scons - name: Compare safety with the previous opendbc timeout-minutes: 20 run: openpilot/selfdrive/pandad/tests/diff_safety.py diff --git a/openpilot/selfdrive/pandad/tests/diff_safety.py b/openpilot/selfdrive/pandad/tests/diff_safety.py index ddb91f0cd4c20d..3148aee81da0e4 100755 --- a/openpilot/selfdrive/pandad/tests/diff_safety.py +++ b/openpilot/selfdrive/pandad/tests/diff_safety.py @@ -86,7 +86,7 @@ def dump_route(args): with open(out / f"{i}.pkl", "wb") as f: pickle.dump((route.route, str(route.car_model), *load(route, tx=True)), f) except Exception as e: - return f"ERROR {route.car_model} - {route.route}: can't load, {e!r}" + return f"{route.car_model} - {route.route}: can't load, {e!r}" return None @@ -167,7 +167,7 @@ def diff_report(dumps: Path, base: Path, new: Path, errors: list[str]) -> int: route, model, mode, _, frames = pickle.load(f) a, b = np.load(base / f"{dump.stem}.npz"), np.load(new / f"{dump.stem}.npz") if "ok" not in b: - errors.append(f"ERROR {model} - {route}: this opendbc has no safety mode {mode}") + errors.append(f"{model} - {route}: this opendbc has no safety mode {mode}") elif "ok" not in a: new_cars.append(f"\nNEW {model} - {route}: master's opendbc has no safety mode {mode}") else: @@ -177,21 +177,12 @@ def diff_report(dumps: Path, base: Path, new: Path, errors: list[str]) -> int: else: n_passed += 1 - icon = "⚠️" if changed or errors else "✅" - lines = [ - "## Safety replay diff report", - "Replays every car test route through master's and this PR's panda safety and compares each RX and TX result and the safety state.", - "Please review any changes carefully to ensure they are expected.\n", - f"{icon} {len(changed)} changed, {n_passed} passed, {len(errors)} errors" + (f", {len(new_cars)} new" if new_cars else ""), - ] - lines += [f"\n{e}" for e in errors] + new_cars - if changed: - lines.append("
Show changes\n\n```") - for route_lines in changed: - lines += route_lines - lines.append("```\n
") - REPORT.write_text("\n".join(lines) + "\n") - print("\n".join(lines)) + from openpilot.selfdrive.test.process_replay.diff_report import format_report + report = format_report("Safety replay diff report", + "Replays every car test route through master's and this PR's panda safety and compares each RX and TX result and the safety state.", + changed, n_passed, errors, new_cars) + REPORT.write_text(report) + print(report) return int(bool(changed or errors)) diff --git a/openpilot/selfdrive/test/process_replay/diff_report.py b/openpilot/selfdrive/test/process_replay/diff_report.py index c6c1f01abd3c36..c932e5301bad0d 100644 --- a/openpilot/selfdrive/test/process_replay/diff_report.py +++ b/openpilot/selfdrive/test/process_replay/diff_report.py @@ -53,6 +53,24 @@ def diff_format(diffs, ref, new, field) -> list[str]: return format_diff(diffs, ref_ts, new_wrapped, field) +def format_report(title: str, description: str, changes: list[list[str]], n_passed: int, errors: list[str], notes: list[str] | None = None) -> str: + icon = "⚠️" if changes else "✅" + lines = [ + f"## {title}", + description, + "Please review any changes carefully to ensure they are expected.\n", + f"{icon} {len(changes)} changed, {n_passed} passed, {len(errors)} errors", + ] + lines += [f"\nERROR {e}" for e in errors] + lines += notes or [] + if changes: + lines.append("
Show changes\n\n```") + for change in changes: + lines += change + lines.append("```\n
") + return "\n".join(lines) + + def diff_report(replay_diffs, segments) -> None: seg_to_plat = {seg: plat for plat, seg in segments} @@ -66,28 +84,18 @@ def diff_report(replay_diffs, segments) -> None: else: with_diffs.append((plat, seg, proc, data)) - icon = "⚠️" if with_diffs else "✅" - lines = [ - "## Process replay diff report", - "Replays driving segments through this PR and compares the behavior to master.", - "Please review any changes carefully to ensure they are expected.\n", - f"{icon} {len(with_diffs)} changed, {n_passed} passed, {len(errors)} errors", - ] - - for plat, seg, proc, err in errors: - lines.append(f"\nERROR {plat} - {seg} [{proc}]: {err}") - - if with_diffs: - lines.append("
Show changes\n\n```") - for plat, seg, proc, (diffs, ref, new) in with_diffs: - lines.append(f"\n{plat} - {seg} [{proc}]") - by_field = defaultdict(list) - for d in diffs: - by_field[d[0]].append(d) - for field, fd in sorted(by_field.items()): - lines.append(f"\n {field} ({len(fd)} diffs)") - lines.extend(diff_format(fd, ref, new, field)) - lines.append("```\n
") + changes = [] + for plat, seg, proc, (diffs, ref, new) in with_diffs: + lines = [f"\n{plat} - {seg} [{proc}]"] + by_field = defaultdict(list) + for d in diffs: + by_field[d[0]].append(d) + for field, fd in sorted(by_field.items()): + lines.append(f"\n {field} ({len(fd)} diffs)") + lines.extend(diff_format(fd, ref, new, field)) + changes.append(lines) + report = format_report("Process replay diff report", "Replays driving segments through this PR and compares the behavior to master.", + changes, n_passed, [f"{plat} - {seg} [{proc}]: {err}" for plat, seg, proc, err in errors]) with open(os.path.join(PROC_REPLAY_DIR, "diff_report.txt"), "w") as f: - f.write("\n".join(lines)) + f.write(report) From 2e99b6e4303f086b348ea934657bd93e8c7c690a Mon Sep 17 00:00:00 2001 From: elkoled Date: Tue, 17 Nov 2026 07:07:09 -0800 Subject: [PATCH 17/21] safety diff report: own paragraph in the comment, fit the comment size --- .github/workflows/diff_report.yaml | 6 +++++- openpilot/selfdrive/pandad/tests/diff_safety.py | 16 +++++++++++++--- .../selfdrive/test/process_replay/diff_report.py | 5 +++-- 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/.github/workflows/diff_report.yaml b/.github/workflows/diff_report.yaml index 471efc2dfcc260..5c6013286c2387 100644 --- a/.github/workflows/diff_report.yaml +++ b/.github/workflows/diff_report.yaml @@ -72,7 +72,11 @@ jobs: # one comment for both reports, the safety one only when this PR changes opendbc - name: Combine reports if: steps.wait.outcome == 'success' - run: cat diff_report.txt diff_safety.txt > comment.md 2>/dev/null; [ -s comment.md ] || rm -f comment.md + run: | + for f in diff_report.txt diff_safety.txt; do + if [ -f "$f" ]; then cat "$f"; printf '\n\n'; fi + done > comment.md + [ -s comment.md ] || rm -f comment.md - name: Comment on PR if: steps.wait.outcome == 'success' && hashFiles('comment.md') != '' uses: thollander/actions-comment-pull-request@24bffb9b452ba05a4f3f77933840a6a841d1b32b diff --git a/openpilot/selfdrive/pandad/tests/diff_safety.py b/openpilot/selfdrive/pandad/tests/diff_safety.py index 3148aee81da0e4..aa1bf238fe21f9 100755 --- a/openpilot/selfdrive/pandad/tests/diff_safety.py +++ b/openpilot/selfdrive/pandad/tests/diff_safety.py @@ -15,6 +15,8 @@ ROOT = Path(__file__).resolve().parents[4] REPORT = ROOT / "diff_safety.txt" +# GitHub comments hold 65536 characters, and this shares one with the process replay report +MAX_REPORT = 50000 DT = 0.01 BUSES = 3 # a TX message's bus, like cereal's can src @@ -178,11 +180,19 @@ def diff_report(dumps: Path, base: Path, new: Path, errors: list[str]) -> int: n_passed += 1 from openpilot.selfdrive.test.process_replay.diff_report import format_report - report = format_report("Safety replay diff report", + def fmt(shown): + return format_report("Safety replay diff report", "Replays every car test route through master's and this PR's panda safety and compares each RX and TX result and the safety state.", - changed, n_passed, errors, new_cars) - REPORT.write_text(report) + shown, n_passed, errors, new_cars, n_changed=len(changed)) + report = fmt(changed) print(report) + # the full report stays in the job log, the comment shows as many routes as fit + n = len(changed) + while n and len(fmt(changed[:n] + [[f"\n(... {len(changed) - n} more routes, see the safety replay job)"]])) > MAX_REPORT: + n -= 1 + if n < len(changed): + report = fmt(changed[:n] + [[f"\n(... {len(changed) - n} more routes, see the safety replay job)"]]) + REPORT.write_text(report) return int(bool(changed or errors)) diff --git a/openpilot/selfdrive/test/process_replay/diff_report.py b/openpilot/selfdrive/test/process_replay/diff_report.py index c932e5301bad0d..7860bd725bd429 100644 --- a/openpilot/selfdrive/test/process_replay/diff_report.py +++ b/openpilot/selfdrive/test/process_replay/diff_report.py @@ -53,13 +53,14 @@ def diff_format(diffs, ref, new, field) -> list[str]: return format_diff(diffs, ref_ts, new_wrapped, field) -def format_report(title: str, description: str, changes: list[list[str]], n_passed: int, errors: list[str], notes: list[str] | None = None) -> str: +def format_report(title: str, description: str, changes: list[list[str]], n_passed: int, errors: list[str], notes: list[str] | None = None, + n_changed: int | None = None) -> str: icon = "⚠️" if changes else "✅" lines = [ f"## {title}", description, "Please review any changes carefully to ensure they are expected.\n", - f"{icon} {len(changes)} changed, {n_passed} passed, {len(errors)} errors", + f"{icon} {len(changes) if n_changed is None else n_changed} changed, {n_passed} passed, {len(errors)} errors", ] lines += [f"\nERROR {e}" for e in errors] lines += notes or [] From 4e78808ed6e7b83917bc9ff6c36af05bfeb58680 Mon Sep 17 00:00:00 2001 From: elkoled Date: Tue, 17 Nov 2026 07:18:36 -0800 Subject: [PATCH 18/21] diff_safety: compare the whole safety state struct, a fresh libsafety per route --- .../selfdrive/pandad/tests/diff_safety.py | 86 ++++++++++++------- 1 file changed, 56 insertions(+), 30 deletions(-) diff --git a/openpilot/selfdrive/pandad/tests/diff_safety.py b/openpilot/selfdrive/pandad/tests/diff_safety.py index aa1bf238fe21f9..df3d0fafaa72d4 100755 --- a/openpilot/selfdrive/pandad/tests/diff_safety.py +++ b/openpilot/selfdrive/pandad/tests/diff_safety.py @@ -10,6 +10,7 @@ from collections import Counter from concurrent.futures import ProcessPoolExecutor from pathlib import Path +from typing import Any import numpy as np @@ -21,13 +22,7 @@ BUSES = 3 # a TX message's bus, like cereal's can src TX = 128 -STATE = ("get_controls_allowed", "get_relay_malfunction", "get_gas_pressed_prev", "get_brake_pressed_prev", "get_regen_braking_prev", - "get_steering_disengage_prev", "get_cruise_engaged_prev", "get_vehicle_moving", "get_acc_main_on", "get_torque_meas_min", - "get_torque_meas_max", "get_torque_driver_min", "get_torque_driver_max", "get_angle_meas_min", "get_angle_meas_max", - "get_curvature_meas_min", "get_curvature_meas_max", "get_vehicle_speed_min", "get_vehicle_speed_max", "get_desired_curvature_last", - "get_honda_fwd_brake") -BOOL_STATE = {"controls_allowed", "relay_malfunction", "gas_pressed_prev", "brake_pressed_prev", "regen_braking_prev", "steering_disengage_prev", - "cruise_engaged_prev", "vehicle_moving", "acc_main_on", "honda_fwd_brake"} + def load(route, seconds: float | None = None, tx: bool = False): @@ -56,13 +51,14 @@ def load(route, seconds: float | None = None, tx: bool = False): def replay(mode: int, param: int, frames, on_tick=None): - # the result of each message and the safety state after each 10 ms step + # the result of each message and the whole safety state (safety_a) after each 10 ms step from opendbc.safety.tests.libsafety import libsafety_py safety = libsafety_py.libsafety assert safety.set_safety_hooks(mode, param) == 0 safety.set_alternative_experience(0) # older opendbc names it safety_tick_current_safety_config tick = safety.safety_tick if hasattr(safety, "safety_tick") else safety.safety_tick_current_safety_config + state = libsafety_py.ffi.buffer(libsafety_py.ffi.addressof(safety, "safety_a")) steps = [] for i, frame in enumerate(frames): @@ -78,7 +74,7 @@ def replay(mode: int, param: int, frames, on_tick=None): else: fwd = safety.safety_fwd_hook(bus, addr) results.append((bool(safety.safety_rx_hook(libsafety_py.make_CANPacket(addr, bus, dat))), fwd)) - steps.append((results, tuple(getattr(safety, g)() for g in STATE))) + steps.append((results, state[:])) return steps @@ -92,8 +88,28 @@ def dump_route(args): return None +def state_layout(ffi): + # every field of the safety state with its offset and type, pointers left out since they differ between processes + fields = [] + + def walk(t, path, offset): + if t.kind == "struct": + for name, f in t.fields: + walk(f.type, f"{path}.{name}" if path else name, offset + f.offset) + elif t.kind == "array": + for i in range(t.length): + walk(t.item, f"{path}[{i}]", offset + i * ffi.sizeof(t.item)) + elif t.kind in ("primitive", "enum"): + size = ffi.sizeof(t) + dtype = "?" if t.cname == "_Bool" else f"f{size}" if t.cname in ("float", "double") else f"u{size}" if t.cname.startswith("u") else f"i{size}" + fields.append((path.removeprefix("state."), offset, size, dtype)) + + walk(ffi.typeof("SafetyCopy"), "", 0) + return fields + + def results_route(args): - # every message's result (RX valid or TX allowed), where each RX is forwarded, and the state after each step + # every message's result (RX valid or TX allowed), where each RX is forwarded, and each safety state field after each step path, out = args with open(path, "rb") as f: route, model, mode, param, frames = pickle.load(f) @@ -103,20 +119,21 @@ def results_route(args): return steps = replay(mode, param, frames) results = [r for step_results, _ in steps for r in step_results] - np.savez(out / path.with_suffix(".npz").name, - ok=np.array([r[0] if isinstance(r, tuple) else r for r in results], dtype=bool), - fwd=np.array([r[1] if isinstance(r, tuple) else -2 for r in results], dtype=np.int16), - state=np.array([state for _, state in steps], dtype=np.float64)) + raw = np.frombuffer(b"".join(state for _, state in steps), dtype=np.uint8).reshape(len(steps), -1) + arrays: dict[str, Any] = {f"state:{name}": raw[:, offset:offset + size].copy().view(dtype)[:, 0] + for name, offset, size, dtype in state_layout(libsafety_py.ffi)} + arrays["ok"] = np.array([r[0] if isinstance(r, tuple) else r for r in results], dtype=bool) + arrays["fwd"] = np.array([r[1] if isinstance(r, tuple) else -2 for r in results], dtype=np.int16) + np.savez_compressed(out / path.with_suffix(".npz").name, **arrays) -def pool(): - return ProcessPoolExecutor(os.cpu_count(), mp_context=multiprocessing.get_context("spawn")) +def pool(fresh: bool = False): + # fresh: a process per task, so no safety state carries over from another route + return ProcessPoolExecutor(os.cpu_count(), mp_context=multiprocessing.get_context("spawn"), max_tasks_per_child=1 if fresh else None) -def value(v: float, name: str): - if name in BOOL_STATE: - return bool(v) - return int(v) if v == int(v) else round(v, 3) +def value(v): + return bool(v) if isinstance(v, np.bool_) else v.item() def message_diffs(frames, a, b) -> list[str]: @@ -138,9 +155,13 @@ def message_diffs(frames, a, b) -> list[str]: class StateWrap: - """Adapter so car_diff reads one safety state value""" + """Adapter so car_diff reads one safety state field, dotted names are nested like in a struct""" def __init__(self, name: str, v): - self._d = {name: v} + *parents, leaf = name.split(".") + self._d = d = {} + for p in parents: + d = d.setdefault(p, {}) + d[leaf] = v def to_dict(self) -> dict: return self._d @@ -148,15 +169,20 @@ def to_dict(self) -> dict: def state_diffs(a, b) -> list[str]: from opendbc.car.tests.car_diff import format_diff lines = [] - for k, getter in enumerate(STATE): - name = getter.removeprefix("get_") - old, new = a["state"][:, k], b["state"][:, k] + for key in sorted(a.keys() | b.keys()): + if not key.startswith("state:"): + continue + name = key.removeprefix("state:") + if key not in a or key not in b: + lines.append(f"\n {name}: only in {'master' if key in a else 'this PR'}") + continue + old, new = a[key], b[key] frames = np.flatnonzero(old != new) if not len(frames): continue - diffs = [(name, int(i), (value(old[i], name), value(new[i], name)), int(i * DT * 1e9)) for i in frames] - ref = [(int(i * DT * 1e9), StateWrap(name, value(v, name))) for i, v in enumerate(old)] - states = [StateWrap(name, value(v, name)) for v in new] + diffs = [(name, int(i), (value(old[i]), value(new[i])), int(i * DT * 1e9)) for i in frames] + ref = [(int(i * DT * 1e9), StateWrap(name, value(v))) for i, v in enumerate(old)] + states = [StateWrap(name, value(v)) for v in new] lines.append(f"\n {name} ({len(diffs)} diffs)") lines += format_diff(diffs, ref, states, name) return lines @@ -167,7 +193,7 @@ def diff_report(dumps: Path, base: Path, new: Path, errors: list[str]) -> int: for dump in sorted(dumps.glob("*.pkl"), key=lambda p: int(p.stem)): with open(dump, "rb") as f: route, model, mode, _, frames = pickle.load(f) - a, b = np.load(base / f"{dump.stem}.npz"), np.load(new / f"{dump.stem}.npz") + a, b = dict(np.load(base / f"{dump.stem}.npz")), dict(np.load(new / f"{dump.stem}.npz")) if "ok" not in b: errors.append(f"{model} - {route}: this opendbc has no safety mode {mode}") elif "ok" not in a: @@ -201,7 +227,7 @@ def main() -> int: if sys.argv[1:2] == ["results"]: dumps, out = Path(sys.argv[2]), Path(sys.argv[3]) out.mkdir() - with pool() as p: + with pool(fresh=True) as p: list(p.map(results_route, [(d, out) for d in sorted(dumps.glob("*.pkl"))])) return 0 From 503f19ba19ed11d6a49d118e84160e5bd1375721 Mon Sep 17 00:00:00 2001 From: elkoled Date: Tue, 17 Nov 2026 07:26:56 -0800 Subject: [PATCH 19/21] safety replay: derive constants from their sources --- .../selfdrive/pandad/tests/diff_safety.py | 31 ++++++++++------ .../selfdrive/pandad/tests/safety_replay.py | 35 +++++++++---------- 2 files changed, 38 insertions(+), 28 deletions(-) diff --git a/openpilot/selfdrive/pandad/tests/diff_safety.py b/openpilot/selfdrive/pandad/tests/diff_safety.py index df3d0fafaa72d4..ce05dc084793ca 100755 --- a/openpilot/selfdrive/pandad/tests/diff_safety.py +++ b/openpilot/selfdrive/pandad/tests/diff_safety.py @@ -14,17 +14,16 @@ import numpy as np -ROOT = Path(__file__).resolve().parents[4] -REPORT = ROOT / "diff_safety.txt" +from opendbc.car import DT_CTRL as DT + # GitHub comments hold 65536 characters, and this shares one with the process replay report MAX_REPORT = 50000 -DT = 0.01 +# the panda's CAN buses, a log's can src is bus + 4 * panda BUSES = 3 -# a TX message's bus, like cereal's can src +# the panda marks sent CAN with src + 128 and blocked with src + 192 TX = 128 - def load(route, seconds: float | None = None, tx: bool = False): from opendbc.car.car_helpers import interfaces from opendbc.car.tests.test_models import TestCarModelBase @@ -42,7 +41,7 @@ def load(route, seconds: float | None = None, tx: bool = False): if i >= len(frames): break for m in msgs: - if m.src < 64 and m.src % 4 < BUSES: + if m.src < TX and m.src % 4 < BUSES: frames[i].append((m.address, bytes(m.dat), m.src % 4)) # sent and blocked TX, so a check that allows more shows up too elif tx and m.src >= TX and m.src % 4 < BUSES: @@ -101,7 +100,15 @@ def walk(t, path, offset): walk(t.item, f"{path}[{i}]", offset + i * ffi.sizeof(t.item)) elif t.kind in ("primitive", "enum"): size = ffi.sizeof(t) - dtype = "?" if t.cname == "_Bool" else f"f{size}" if t.cname in ("float", "double") else f"u{size}" if t.cname.startswith("u") else f"i{size}" + if t.cname == "_Bool": + dtype = "?" + else: + # cffi only converts float types with float() + try: + float(ffi.cast(t, 0)) + dtype = f"f{size}" + except TypeError: + dtype = f"i{size}" if int(ffi.cast(t, -1)) < 0 else f"u{size}" fields.append((path.removeprefix("state."), offset, size, dtype)) walk(ffi.typeof("SafetyCopy"), "", 0) @@ -218,7 +225,8 @@ def fmt(shown): n -= 1 if n < len(changed): report = fmt(changed[:n] + [[f"\n(... {len(changed) - n} more routes, see the safety replay job)"]]) - REPORT.write_text(report) + from openpilot.common.basedir import BASEDIR + (Path(BASEDIR) / "diff_safety.txt").write_text(report) return int(bool(changed or errors)) @@ -231,7 +239,10 @@ def main() -> int: list(p.map(results_route, [(d, out) for d in sorted(dumps.glob("*.pkl"))])) return 0 - def git(*args, cwd=ROOT): + from openpilot.common.basedir import BASEDIR + root = Path(BASEDIR) + + def git(*args, cwd=root): return subprocess.check_output(["git", *args], cwd=cwd, text=True).strip() base_ref = sys.argv[1] if len(sys.argv) > 1 else "HEAD^" @@ -243,7 +254,7 @@ def git(*args, cwd=ROOT): from opendbc.car.tests.routes import routes with tempfile.TemporaryDirectory() as tmp: - w, odbc = Path(tmp), ROOT / "opendbc_repo" + w, odbc = Path(tmp), root / "opendbc_repo" git("fetch", "-q", "--depth=1", "origin", base, cwd=odbc) git("worktree", "add", "-q", "--detach", str(w / "base"), base, cwd=odbc) try: diff --git a/openpilot/selfdrive/pandad/tests/safety_replay.py b/openpilot/selfdrive/pandad/tests/safety_replay.py index 85546ffab0fd16..ae85fad2326a21 100755 --- a/openpilot/selfdrive/pandad/tests/safety_replay.py +++ b/openpilot/selfdrive/pandad/tests/safety_replay.py @@ -1,11 +1,13 @@ #!/usr/bin/env python3 # checks the panda's safety against libsafety on the car test routes import hashlib +import re import multiprocessing import os import subprocess import sys import time +import tomllib from collections import Counter from concurrent.futures import ProcessPoolExecutor from concurrent.futures.process import BrokenProcessPool @@ -23,26 +25,22 @@ STEADY = 2 # frames the station's CAN lanes can start apart SKEW = 3 -# master's passing runs, restored with the alfred cache -PASSED = Path("/data/scons_cache/safety_replay") +# master's passing runs go in alfred's first cache dir, which jobs get back from master's last good job +PASSED = Path(tomllib.loads((Path(BASEDIR) / "alfred.toml").read_text())["cache"][0]) / "safety_replay" # a pass is reused for a day, so the rack itself is still checked daily PASS_TTL = 24 * 3600 -BUILD_OUTPUTS = {".so", ".o", ".os", ".a", ".pyc", ".gcno", ".gcda"} def inputs() -> str: - # everything the result depends on, except the rack: the firmware, opendbc, the panda library, this test, the OS and its compiler - root = Path(BASEDIR) - files = sorted((root / "panda/board/obj").glob("*.bin.signed")) - for d in ("opendbc_repo/opendbc", "panda/python"): - files += sorted(f for f in (root / d).rglob("*") if f.is_file() and f.suffix not in BUILD_OUTPUTS and "__pycache__" not in f.parts) - files += [Path(__file__).resolve(), Path(__file__).with_name("diff_safety.py").resolve(), root / "openpilot/selfdrive/pandad/pandad.py"] - h = hashlib.sha256() - for f in files: - h.update(str(f.relative_to(root)).encode() + b"\0" + f.read_bytes()) - h.update(f"{HARDWARE.get_os_version()}".encode() + subprocess.check_output(["cc", "--version"])) - h.update(f"{NUM_JOBS} {JOB_ID}".encode()) - return h.hexdigest()[:16] + # everything the result depends on, except the rack: git's ids of opendbc, panda and this test's dir, and the OS that builds them + ids = subprocess.check_output(["git", "rev-parse", "HEAD:opendbc_repo", "HEAD:panda", "HEAD:openpilot/selfdrive/pandad"], cwd=BASEDIR, text=True) + return hashlib.sha256(f"{ids} {HARDWARE.get_os_version()} {NUM_JOBS} {JOB_ID}".encode()).hexdigest()[:16] + + +def relay_malfunction_fault() -> int: + # the fault bit as the panda firmware defines it + sys_h = (Path(BASEDIR) / "panda/board/sys/sys.h").read_text() + return 1 << int(re.search(r"#define FAULT_RELAY_MALFUNCTION\s+\(1UL << (\d+)\)", sys_h).group(1)) def passed_age(key: str) -> float | None: @@ -155,7 +153,7 @@ def played(p, up, mode, param, frames): if fd & {0, 2}: fd |= {0, 2} for b in fd: - up.set_can_data_speed_kbps(b, 2000) + up.set_can_data_speed_kbps(b, h0[b]["can_data_speed"]) # pad the end so the UP doesn't loop the route up.can_replay(frames + [[]] * int(60 / DT)) start, checks_invalid = time.monotonic(), False @@ -166,13 +164,14 @@ def played(p, up, mode, param, frames): checks_invalid |= p.health()["safety_rx_checks_invalid"] time.sleep(0.2) h, h1 = p.health(), [p.can_health(c) for c in cans] + relay = relay_malfunction_fault() up.reset() up.set_ignition(True) return {"rx": [h1[b]["total_rx_cnt"] - h0[b]["total_rx_cnt"] for b in range(BUSES)], "fwd": [h1[b]["total_fwd_cnt"] - h0[b]["total_fwd_cnt"] for b in range(BUSES)], - "rx_invalid": h["safety_rx_invalid"], "relay_malfunction": bool(h["faults"] & 1), "rx_checks_invalid": checks_invalid, + "rx_invalid": h["safety_rx_invalid"], "relay_malfunction": bool(h["faults"] & relay), "rx_checks_invalid": checks_invalid, "safety": [h["safety_mode"], h["safety_param"]], - "faults": h["faults"] & ~hs0["faults"] & ~1, + "faults": h["faults"] & ~hs0["faults"] & ~relay, # TX only, RX overflows without pandad "tx_buffer_overflow": h["tx_buffer_overflow"] - hs0["tx_buffer_overflow"], "heartbeat_disabled": h["heartbeat_disabled"], "harness_changed": h["car_harness_status"] != hs0["car_harness_status"], From 9080819b53c4072441f89046bb3b67af74685476 Mon Sep 17 00:00:00 2001 From: elkoled Date: Tue, 17 Nov 2026 07:40:25 -0800 Subject: [PATCH 20/21] safety replay CI: skip when opendbc is unchanged, no build, one libsafety build per side --- .github/workflows/tests.yaml | 9 ++--- .../selfdrive/pandad/tests/diff_safety.py | 34 +++++++++++++------ openpilot/selfdrive/test/diff_format.py | 20 +++++++++++ .../test/process_replay/diff_report.py | 20 +---------- 4 files changed, 50 insertions(+), 33 deletions(-) create mode 100644 openpilot/selfdrive/test/diff_format.py diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 1324a5551fed9e..feb24413a27f99 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -187,20 +187,21 @@ jobs: steps: - uses: actions/checkout@v7 with: - submodules: true # the parent is master, or the previous master commit on a push fetch-depth: 2 - run: ./tools/op.sh setup + - name: Check for an opendbc change + id: opendbc + run: openpilot/selfdrive/pandad/tests/diff_safety.py check >> "$GITHUB_OUTPUT" - name: Cache the test routes + if: steps.opendbc.outputs.changed == 'true' uses: actions/cache@v6 with: path: /tmp/comma_download_cache key: safety-replay-${{ hashFiles('opendbc_repo/opendbc/car/tests/routes.py') }} restore-keys: safety-replay- - # the report reuses process replay's diff report layout - - name: Build openpilot - run: scons - name: Compare safety with the previous opendbc + if: steps.opendbc.outputs.changed == 'true' timeout-minutes: 20 run: openpilot/selfdrive/pandad/tests/diff_safety.py - name: Upload diff report diff --git a/openpilot/selfdrive/pandad/tests/diff_safety.py b/openpilot/selfdrive/pandad/tests/diff_safety.py index ce05dc084793ca..199fb25d0de9e9 100755 --- a/openpilot/selfdrive/pandad/tests/diff_safety.py +++ b/openpilot/selfdrive/pandad/tests/diff_safety.py @@ -1,9 +1,11 @@ #!/usr/bin/env python3 # replays the car test routes through BASE's and this opendbc's safety, reports every changed RX and TX result and safety state value -# diff_safety.py [BASE] BASE defaults to HEAD^, which is master on a PR's merge commit +# diff_safety.py [BASE] BASE defaults to HEAD^, which is master on a PR's merge commit +# diff_safety.py check [BASE] prints changed=true when opendbc differs from BASE's import multiprocessing import os import pickle +import shutil import subprocess import sys import tempfile @@ -117,10 +119,16 @@ def walk(t, path, offset): def results_route(args): # every message's result (RX valid or TX allowed), where each RX is forwarded, and each safety state field after each step - path, out = args + path, out, build = args with open(path, "rb") as f: route, model, mode, param, frames = pickle.load(f) from opendbc.safety.tests.libsafety import libsafety_py + # a copy of the build is a fresh library, no safety state carries over from another route + fd, copy = tempfile.mkstemp(suffix=".so") + os.close(fd) + shutil.copy(build, copy) + libsafety_py.load(copy) + os.unlink(copy) if libsafety_py.libsafety.set_safety_hooks(mode, param) != 0: np.savez(out / path.with_suffix(".npz").name, mode=mode) return @@ -134,9 +142,8 @@ def results_route(args): np.savez_compressed(out / path.with_suffix(".npz").name, **arrays) -def pool(fresh: bool = False): - # fresh: a process per task, so no safety state carries over from another route - return ProcessPoolExecutor(os.cpu_count(), mp_context=multiprocessing.get_context("spawn"), max_tasks_per_child=1 if fresh else None) +def pool(): + return ProcessPoolExecutor(os.cpu_count(), mp_context=multiprocessing.get_context("spawn")) def value(v): @@ -212,7 +219,7 @@ def diff_report(dumps: Path, base: Path, new: Path, errors: list[str]) -> int: else: n_passed += 1 - from openpilot.selfdrive.test.process_replay.diff_report import format_report + from openpilot.selfdrive.test.diff_format import format_report def fmt(shown): return format_report("Safety replay diff report", "Replays every car test route through master's and this PR's panda safety and compares each RX and TX result and the safety state.", @@ -235,8 +242,10 @@ def main() -> int: if sys.argv[1:2] == ["results"]: dumps, out = Path(sys.argv[2]), Path(sys.argv[3]) out.mkdir() - with pool(fresh=True) as p: - list(p.map(results_route, [(d, out) for d in sorted(dumps.glob("*.pkl"))])) + from opendbc.safety.tests.libsafety import libsafety_py + build = libsafety_py._build_libsafety() + with pool() as p: + list(p.map(results_route, [(d, out, build) for d in sorted(dumps.glob("*.pkl"))])) return 0 from openpilot.common.basedir import BASEDIR @@ -245,8 +254,13 @@ def main() -> int: def git(*args, cwd=root): return subprocess.check_output(["git", *args], cwd=cwd, text=True).strip() - base_ref = sys.argv[1] if len(sys.argv) > 1 else "HEAD^" - base, new = git("rev-parse", f"{base_ref}:opendbc_repo"), git("rev-parse", "HEAD:opendbc_repo") + check = sys.argv[1:2] == ["check"] + base_ref = (sys.argv[2:] if check else sys.argv[1:]) or ["HEAD^"] + base, new = git("rev-parse", f"{base_ref[0]}:opendbc_repo"), git("rev-parse", "HEAD:opendbc_repo") + # for a GitHub step's outputs + if check: + print(f"changed={str(base != new).lower()}") + return 0 if base == new: print(f"opendbc is unchanged ({new}), nothing to compare") return 0 diff --git a/openpilot/selfdrive/test/diff_format.py b/openpilot/selfdrive/test/diff_format.py new file mode 100644 index 00000000000000..806867fed6cf1e --- /dev/null +++ b/openpilot/selfdrive/test/diff_format.py @@ -0,0 +1,20 @@ +# the layout of the diff report comments, light to import: CI jobs without a build use it + + +def format_report(title: str, description: str, changes: list[list[str]], n_passed: int, errors: list[str], notes: list[str] | None = None, + n_changed: int | None = None) -> str: + icon = "⚠️" if changes else "✅" + lines = [ + f"## {title}", + description, + "Please review any changes carefully to ensure they are expected.\n", + f"{icon} {len(changes) if n_changed is None else n_changed} changed, {n_passed} passed, {len(errors)} errors", + ] + lines += [f"\nERROR {e}" for e in errors] + lines += notes or [] + if changes: + lines.append("
Show changes\n\n```") + for change in changes: + lines += change + lines.append("```\n
") + return "\n".join(lines) diff --git a/openpilot/selfdrive/test/process_replay/diff_report.py b/openpilot/selfdrive/test/process_replay/diff_report.py index 7860bd725bd429..f91b61ea541d84 100644 --- a/openpilot/selfdrive/test/process_replay/diff_report.py +++ b/openpilot/selfdrive/test/process_replay/diff_report.py @@ -1,6 +1,7 @@ import os from collections import defaultdict from opendbc.car.tests.car_diff import format_diff, format_numeric_diffs +from openpilot.selfdrive.test.diff_format import format_report from openpilot.selfdrive.test.process_replay.compare_logs import compare_logs from openpilot.selfdrive.test.process_replay.process_replay import PROC_REPLAY_DIR @@ -53,25 +54,6 @@ def diff_format(diffs, ref, new, field) -> list[str]: return format_diff(diffs, ref_ts, new_wrapped, field) -def format_report(title: str, description: str, changes: list[list[str]], n_passed: int, errors: list[str], notes: list[str] | None = None, - n_changed: int | None = None) -> str: - icon = "⚠️" if changes else "✅" - lines = [ - f"## {title}", - description, - "Please review any changes carefully to ensure they are expected.\n", - f"{icon} {len(changes) if n_changed is None else n_changed} changed, {n_passed} passed, {len(errors)} errors", - ] - lines += [f"\nERROR {e}" for e in errors] - lines += notes or [] - if changes: - lines.append("
Show changes\n\n```") - for change in changes: - lines += change - lines.append("```\n
") - return "\n".join(lines) - - def diff_report(replay_diffs, segments) -> None: seg_to_plat = {seg: plat for plat, seg in segments} From c579e090690935c8ff69a1a2f2e5fce7a991c436 Mon Sep 17 00:00:00 2001 From: elkoled Date: Tue, 17 Nov 2026 07:54:18 -0800 Subject: [PATCH 21/21] diff_safety: full report only in the artifact --- .github/workflows/tests.yaml | 4 +- .../selfdrive/pandad/tests/diff_safety.py | 48 ++++++++++++------- 2 files changed, 33 insertions(+), 19 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index feb24413a27f99..6e0ef9b5b144c7 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -209,7 +209,9 @@ jobs: uses: actions/upload-artifact@v7 with: name: diff_safety_${{ github.event.number }} - path: diff_safety.txt + path: | + diff_safety.txt + diff_safety_full.txt if-no-files-found: ignore simulator_driving: diff --git a/openpilot/selfdrive/pandad/tests/diff_safety.py b/openpilot/selfdrive/pandad/tests/diff_safety.py index 199fb25d0de9e9..1d8b1191c8bb80 100755 --- a/openpilot/selfdrive/pandad/tests/diff_safety.py +++ b/openpilot/selfdrive/pandad/tests/diff_safety.py @@ -180,8 +180,9 @@ def to_dict(self) -> dict: return self._d -def state_diffs(a, b) -> list[str]: - from opendbc.car.tests.car_diff import format_diff +def state_diffs(a, b, full: bool = False) -> list[str]: + # full: every changed frame instead of the first ones, for the report in the artifact + from opendbc.car.tests.car_diff import format_diff, format_numeric_diffs lines = [] for key in sorted(a.keys() | b.keys()): if not key.startswith("state:"): @@ -198,12 +199,16 @@ def state_diffs(a, b) -> list[str]: ref = [(int(i * DT * 1e9), StateWrap(name, value(v))) for i, v in enumerate(old)] states = [StateWrap(name, value(v)) for v in new] lines.append(f"\n {name} ({len(diffs)} diffs)") - lines += format_diff(diffs, ref, states, name) + if full and not isinstance(diffs[0][2][0], bool): + for i in range(0, len(diffs), 10): + lines += format_numeric_diffs(diffs[i:i + 10]) + else: + lines += format_diff(diffs, ref, states, name) return lines def diff_report(dumps: Path, base: Path, new: Path, errors: list[str]) -> int: - changed, new_cars, n_passed = [], [], 0 + changed, changed_full, new_cars, n_passed = [], [], [], 0 for dump in sorted(dumps.glob("*.pkl"), key=lambda p: int(p.stem)): with open(dump, "rb") as f: route, model, mode, _, frames = pickle.load(f) @@ -213,26 +218,33 @@ def diff_report(dumps: Path, base: Path, new: Path, errors: list[str]) -> int: elif "ok" not in a: new_cars.append(f"\nNEW {model} - {route}: master's opendbc has no safety mode {mode}") else: - lines = message_diffs(frames, a, b) + state_diffs(a, b) - if lines: - changed.append([f"\n{model} - {route}"] + lines) + messages, states = message_diffs(frames, a, b), state_diffs(a, b) + if messages or states: + changed.append([f"\n{model} - {route}"] + messages + states) + changed_full.append([f"\n{model} - {route}"] + messages + state_diffs(a, b, full=True)) else: n_passed += 1 + from openpilot.common.basedir import BASEDIR from openpilot.selfdrive.test.diff_format import format_report - def fmt(shown): - return format_report("Safety replay diff report", - "Replays every car test route through master's and this PR's panda safety and compares each RX and TX result and the safety state.", - shown, n_passed, errors, new_cars, n_changed=len(changed)) - report = fmt(changed) - print(report) - # the full report stays in the job log, the comment shows as many routes as fit + + def fmt(shown, notes): + return format_report("Safety replay diff report", "Replays every car test route through this PR's panda safety and compares the behavior to master.", + shown, n_passed, errors, new_cars + notes, n_changed=len(changed)) + + (Path(BASEDIR) / "diff_safety_full.txt").write_text(fmt(changed_full, [])) + + # the comment shows as many routes as fit, the full report is in the job's artifact + notes = [] + if changed and "GITHUB_RUN_ID" in os.environ: + run = f"{os.environ['GITHUB_SERVER_URL']}/{os.environ['GITHUB_REPOSITORY']}/actions/runs/{os.environ['GITHUB_RUN_ID']}" + notes.append(f"\nEvery changed frame: diff_safety_full.txt in {run}") n = len(changed) - while n and len(fmt(changed[:n] + [[f"\n(... {len(changed) - n} more routes, see the safety replay job)"]])) > MAX_REPORT: + while n and len(fmt(changed[:n] + [[f"\n(... {len(changed) - n} more routes)"]], notes)) > MAX_REPORT: n -= 1 - if n < len(changed): - report = fmt(changed[:n] + [[f"\n(... {len(changed) - n} more routes, see the safety replay job)"]]) - from openpilot.common.basedir import BASEDIR + shown = changed if n == len(changed) else changed[:n] + [[f"\n(... {len(changed) - n} more routes)"]] + report = fmt(shown, notes) + print(report) (Path(BASEDIR) / "diff_safety.txt").write_text(report) return int(bool(changed or errors))