diff --git a/.github/workflows/diff_report.yaml b/.github/workflows/diff_report.yaml
index 202bc0ec7934af..5c6013286c2387 100644
--- a/.github/workflows/diff_report.yaml
+++ b/.github/workflows/diff_report.yaml
@@ -48,11 +48,40 @@ jobs:
run-id: ${{ steps.tests-run.outputs.run-id }}
name: diff_report_${{ github.event.number }}
path: .
+ - name: Wait for safety replay
+ if: steps.wait.outcome == 'success'
+ id: wait-safety
+ continue-on-error: true
+ uses: lewagon/wait-on-check-action@1d57e2c51a58d812d2765e036a028b6bdb5a6154
+ with:
+ ref: ${{ github.event.pull_request.head.sha }}
+ check-name: safety replay
+ repo-token: ${{ secrets.GITHUB_TOKEN }}
+ allowed-conclusions: success,failure
+ wait-interval: 20
+ - name: Download safety diff
+ if: steps.wait-safety.outcome == 'success'
+ continue-on-error: true
+ uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c
+ with:
+ github-token: ${{ secrets.GITHUB_TOKEN }}
+ repository: ${{ github.repository }}
+ run-id: ${{ steps.tests-run.outputs.run-id }}
+ name: diff_safety_${{ github.event.number }}
+ path: .
+ # one comment for both reports, the safety one only when this PR changes opendbc
+ - name: Combine reports
+ if: steps.wait.outcome == 'success'
+ run: |
+ for f in diff_report.txt diff_safety.txt; do
+ if [ -f "$f" ]; then cat "$f"; printf '\n\n'; fi
+ done > comment.md
+ [ -s comment.md ] || rm -f comment.md
- name: Comment on PR
- if: steps.wait.outcome == 'success' && hashFiles('diff_report.txt') != ''
+ if: steps.wait.outcome == 'success' && hashFiles('comment.md') != ''
uses: thollander/actions-comment-pull-request@24bffb9b452ba05a4f3f77933840a6a841d1b32b
with:
- file-path: diff_report.txt
+ file-path: comment.md
comment-tag: diff_report
pr-number: ${{ github.event.number }}
github-token: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml
index 4777bc9c427c30..6e0ef9b5b144c7 100644
--- a/.github/workflows/tests.yaml
+++ b/.github/workflows/tests.yaml
@@ -174,6 +174,46 @@ jobs:
max_attempts: 3
command: cd ${{ github.workspace }}/ci-artifacts && git push origin process-replay --force
+ safety_replay:
+ name: safety replay
+ runs-on: ${{
+ (github.repository == 'commaai/openpilot') &&
+ ((github.event_name != 'pull_request') ||
+ (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
+ && fromJSON('["namespace-profile-amd64-8x16"]')
+ || fromJSON('["ubuntu-24.04"]') }}
+ env:
+ COMMA_CACHE: /tmp/comma_download_cache
+ steps:
+ - uses: actions/checkout@v7
+ with:
+ # the parent is master, or the previous master commit on a push
+ fetch-depth: 2
+ - run: ./tools/op.sh setup
+ - name: Check for an opendbc change
+ id: opendbc
+ run: openpilot/selfdrive/pandad/tests/diff_safety.py check >> "$GITHUB_OUTPUT"
+ - name: Cache the test routes
+ if: steps.opendbc.outputs.changed == 'true'
+ uses: actions/cache@v6
+ with:
+ path: /tmp/comma_download_cache
+ key: safety-replay-${{ hashFiles('opendbc_repo/opendbc/car/tests/routes.py') }}
+ restore-keys: safety-replay-
+ - name: Compare safety with the previous opendbc
+ if: steps.opendbc.outputs.changed == 'true'
+ timeout-minutes: 20
+ run: openpilot/selfdrive/pandad/tests/diff_safety.py
+ - name: Upload diff report
+ if: always() && github.event_name == 'pull_request'
+ uses: actions/upload-artifact@v7
+ with:
+ name: diff_safety_${{ github.event.number }}
+ path: |
+ diff_safety.txt
+ diff_safety_full.txt
+ if-no-files-found: ignore
+
simulator_driving:
name: simulator driving
runs-on: ${{
diff --git a/alfred.toml b/alfred.toml
index 9ba6ebeda2cd9f..3b96e10a08c730 100644
--- a/alfred.toml
+++ b/alfred.toml
@@ -23,6 +23,10 @@ for p in msgq_repo/msgq opendbc_repo/opendbc rednose_repo/rednose teleoprtc_repo
tools/op.sh build
"""
+[steps.safety-replay]
+name = "Safety replay"
+run = "./openpilot/selfdrive/pandad/tests/safety_replay.py"
+
[steps.flash-chestnut]
name = "Flash Chestnut"
run = "sudo python3 openpilot/system/hardware/chestnut/flash.py"
@@ -117,6 +121,55 @@ uses = "build"
name = "Sensord"
run = "./openpilot/system/sensord/tests/test_sensord.py"
+# each job replays an eighth of the routes, unless master passed the same inputs in the last day
+[jobs.safety_0]
+runs-on = "up-can"
+timeout-minutes = 25
+env = { NUM_JOBS = "8", JOB_ID = "0" }
+steps = [{ uses = "build" }, { uses = "safety-replay" }]
+
+[jobs.safety_1]
+runs-on = "up-can"
+timeout-minutes = 25
+env = { NUM_JOBS = "8", JOB_ID = "1" }
+steps = [{ uses = "build" }, { uses = "safety-replay" }]
+
+[jobs.safety_2]
+runs-on = "up-can"
+timeout-minutes = 25
+env = { NUM_JOBS = "8", JOB_ID = "2" }
+steps = [{ uses = "build" }, { uses = "safety-replay" }]
+
+[jobs.safety_3]
+runs-on = "up-can"
+timeout-minutes = 25
+env = { NUM_JOBS = "8", JOB_ID = "3" }
+steps = [{ uses = "build" }, { uses = "safety-replay" }]
+
+[jobs.safety_4]
+runs-on = "up-can"
+timeout-minutes = 25
+env = { NUM_JOBS = "8", JOB_ID = "4" }
+steps = [{ uses = "build" }, { uses = "safety-replay" }]
+
+[jobs.safety_5]
+runs-on = "up-can"
+timeout-minutes = 25
+env = { NUM_JOBS = "8", JOB_ID = "5" }
+steps = [{ uses = "build" }, { uses = "safety-replay" }]
+
+[jobs.safety_6]
+runs-on = "up-can"
+timeout-minutes = 25
+env = { NUM_JOBS = "8", JOB_ID = "6" }
+steps = [{ uses = "build" }, { uses = "safety-replay" }]
+
+[jobs.safety_7]
+runs-on = "up-can"
+timeout-minutes = 25
+env = { NUM_JOBS = "8", JOB_ID = "7" }
+steps = [{ uses = "build" }, { uses = "safety-replay" }]
+
# a comma four with a Chestnut on its aux port, which runs the big model
[jobs.chestnut]
runs-on = "chestnut"
diff --git a/opendbc_repo b/opendbc_repo
index 27fa1c5d8eed32..ec18ed089a32e4 160000
--- a/opendbc_repo
+++ b/opendbc_repo
@@ -1 +1 @@
-Subproject commit 27fa1c5d8eed3238a0a78c52ff5abb6cdc807bd2
+Subproject commit ec18ed089a32e4bdc7e259e1f2d0725baf73a48a
diff --git a/openpilot/selfdrive/pandad/tests/diff_safety.py b/openpilot/selfdrive/pandad/tests/diff_safety.py
new file mode 100755
index 00000000000000..1d8b1191c8bb80
--- /dev/null
+++ b/openpilot/selfdrive/pandad/tests/diff_safety.py
@@ -0,0 +1,298 @@
+#!/usr/bin/env python3
+# replays the car test routes through BASE's and this opendbc's safety, reports every changed RX and TX result and safety state value
+# diff_safety.py [BASE] BASE defaults to HEAD^, which is master on a PR's merge commit
+# diff_safety.py check [BASE] prints changed=true when opendbc differs from BASE's
+import multiprocessing
+import os
+import pickle
+import shutil
+import subprocess
+import sys
+import tempfile
+from collections import Counter
+from concurrent.futures import ProcessPoolExecutor
+from pathlib import Path
+from typing import Any
+
+import numpy as np
+
+from opendbc.car import DT_CTRL as DT
+
+# GitHub comments hold 65536 characters, and this shares one with the process replay report
+MAX_REPORT = 50000
+# the panda's CAN buses, a log's can src is bus + 4 * panda
+BUSES = 3
+# the panda marks sent CAN with src + 128 and blocked with src + 192
+TX = 128
+
+
+def load(route, seconds: float | None = None, tx: bool = False):
+ from opendbc.car.car_helpers import interfaces
+ from opendbc.car.tests.test_models import TestCarModelBase
+
+ R = type("R", (TestCarModelBase,), {"test_route": route, "platform": route.car_model})
+ car_fw, can_msgs, alpha_long = R.get_testing_data(route)
+ CP = interfaces[R.platform].get_params(R.platform, R.fingerprint, car_fw, alpha_long, False, docs=False)
+ cfg = CP.safetyConfigs[-1]
+
+ t0 = can_msgs[0][0]
+ steps = int((can_msgs[-1][0] - t0) / 1e9 / DT) + 1 if seconds is None else int(seconds / DT)
+ frames = [[] for _ in range(steps)]
+ for t, msgs in can_msgs:
+ i = int((t - t0) / 1e9 / DT)
+ if i >= len(frames):
+ break
+ for m in msgs:
+ if m.src < TX and m.src % 4 < BUSES:
+ frames[i].append((m.address, bytes(m.dat), m.src % 4))
+ # sent and blocked TX, so a check that allows more shows up too
+ elif tx and m.src >= TX and m.src % 4 < BUSES:
+ frames[i].append((m.address, bytes(m.dat), TX + m.src % 4))
+ return cfg.safetyModel.raw, cfg.safetyParam, frames
+
+
+def replay(mode: int, param: int, frames, on_tick=None):
+ # the result of each message and the whole safety state (safety_a) after each 10 ms step
+ from opendbc.safety.tests.libsafety import libsafety_py
+ safety = libsafety_py.libsafety
+ assert safety.set_safety_hooks(mode, param) == 0
+ safety.set_alternative_experience(0)
+ # older opendbc names it safety_tick_current_safety_config
+ tick = safety.safety_tick if hasattr(safety, "safety_tick") else safety.safety_tick_current_safety_config
+ state = libsafety_py.ffi.buffer(libsafety_py.ffi.addressof(safety, "safety_a"))
+
+ steps = []
+ for i, frame in enumerate(frames):
+ safety.set_timer(int(i * DT * 1e6))
+ if i % int(1 / DT) == 0:
+ tick()
+ if on_tick is not None:
+ on_tick(i, safety)
+ results = []
+ for addr, dat, bus in frame:
+ if bus >= TX:
+ results.append(bool(safety.safety_tx_hook(libsafety_py.make_CANPacket(addr, bus - TX, dat))))
+ else:
+ fwd = safety.safety_fwd_hook(bus, addr)
+ results.append((bool(safety.safety_rx_hook(libsafety_py.make_CANPacket(addr, bus, dat))), fwd))
+ steps.append((results, state[:]))
+ return steps
+
+
+def dump_route(args):
+ i, route, out = args
+ try:
+ with open(out / f"{i}.pkl", "wb") as f:
+ pickle.dump((route.route, str(route.car_model), *load(route, tx=True)), f)
+ except Exception as e:
+ return f"{route.car_model} - {route.route}: can't load, {e!r}"
+ return None
+
+
+def state_layout(ffi):
+ # every field of the safety state with its offset and type, pointers left out since they differ between processes
+ fields = []
+
+ def walk(t, path, offset):
+ if t.kind == "struct":
+ for name, f in t.fields:
+ walk(f.type, f"{path}.{name}" if path else name, offset + f.offset)
+ elif t.kind == "array":
+ for i in range(t.length):
+ walk(t.item, f"{path}[{i}]", offset + i * ffi.sizeof(t.item))
+ elif t.kind in ("primitive", "enum"):
+ size = ffi.sizeof(t)
+ if t.cname == "_Bool":
+ dtype = "?"
+ else:
+ # cffi only converts float types with float()
+ try:
+ float(ffi.cast(t, 0))
+ dtype = f"f{size}"
+ except TypeError:
+ dtype = f"i{size}" if int(ffi.cast(t, -1)) < 0 else f"u{size}"
+ fields.append((path.removeprefix("state."), offset, size, dtype))
+
+ walk(ffi.typeof("SafetyCopy"), "", 0)
+ return fields
+
+
+def results_route(args):
+ # every message's result (RX valid or TX allowed), where each RX is forwarded, and each safety state field after each step
+ path, out, build = args
+ with open(path, "rb") as f:
+ route, model, mode, param, frames = pickle.load(f)
+ from opendbc.safety.tests.libsafety import libsafety_py
+ # a copy of the build is a fresh library, no safety state carries over from another route
+ fd, copy = tempfile.mkstemp(suffix=".so")
+ os.close(fd)
+ shutil.copy(build, copy)
+ libsafety_py.load(copy)
+ os.unlink(copy)
+ if libsafety_py.libsafety.set_safety_hooks(mode, param) != 0:
+ np.savez(out / path.with_suffix(".npz").name, mode=mode)
+ return
+ steps = replay(mode, param, frames)
+ results = [r for step_results, _ in steps for r in step_results]
+ raw = np.frombuffer(b"".join(state for _, state in steps), dtype=np.uint8).reshape(len(steps), -1)
+ arrays: dict[str, Any] = {f"state:{name}": raw[:, offset:offset + size].copy().view(dtype)[:, 0]
+ for name, offset, size, dtype in state_layout(libsafety_py.ffi)}
+ arrays["ok"] = np.array([r[0] if isinstance(r, tuple) else r for r in results], dtype=bool)
+ arrays["fwd"] = np.array([r[1] if isinstance(r, tuple) else -2 for r in results], dtype=np.int16)
+ np.savez_compressed(out / path.with_suffix(".npz").name, **arrays)
+
+
+def pool():
+ return ProcessPoolExecutor(os.cpu_count(), mp_context=multiprocessing.get_context("spawn"))
+
+
+def value(v):
+ return bool(v) if isinstance(v, np.bool_) else v.item()
+
+
+def message_diffs(frames, a, b) -> list[str]:
+ # RX valid, RX forward and TX allowed, grouped by message and change
+ events = [(i, addr, bus) for i, frame in enumerate(frames) for addr, _, bus in frame]
+ changes = Counter()
+ first = {}
+ for k in np.flatnonzero((a["ok"] != b["ok"]) | (a["fwd"] != b["fwd"])):
+ i, addr, bus = events[k]
+ if bus >= TX:
+ key = (f"TX 0x{addr:x} bus {bus - TX} allowed", bool(a["ok"][k]), bool(b["ok"][k]))
+ elif a["ok"][k] != b["ok"][k]:
+ key = (f"RX 0x{addr:x} bus {bus} valid", bool(a["ok"][k]), bool(b["ok"][k]))
+ else:
+ key = (f"RX 0x{addr:x} bus {bus} forward", int(a["fwd"][k]), int(b["fwd"][k]))
+ changes[key] += 1
+ first.setdefault(key, i)
+ return [f"\n {what}: {old} -> {new} ({n} msgs, from frame {first[(what, old, new)]})" for (what, old, new), n in sorted(changes.items())]
+
+
+class StateWrap:
+ """Adapter so car_diff reads one safety state field, dotted names are nested like in a struct"""
+ def __init__(self, name: str, v):
+ *parents, leaf = name.split(".")
+ self._d = d = {}
+ for p in parents:
+ d = d.setdefault(p, {})
+ d[leaf] = v
+ def to_dict(self) -> dict:
+ return self._d
+
+
+def state_diffs(a, b, full: bool = False) -> list[str]:
+ # full: every changed frame instead of the first ones, for the report in the artifact
+ from opendbc.car.tests.car_diff import format_diff, format_numeric_diffs
+ lines = []
+ for key in sorted(a.keys() | b.keys()):
+ if not key.startswith("state:"):
+ continue
+ name = key.removeprefix("state:")
+ if key not in a or key not in b:
+ lines.append(f"\n {name}: only in {'master' if key in a else 'this PR'}")
+ continue
+ old, new = a[key], b[key]
+ frames = np.flatnonzero(old != new)
+ if not len(frames):
+ continue
+ diffs = [(name, int(i), (value(old[i]), value(new[i])), int(i * DT * 1e9)) for i in frames]
+ ref = [(int(i * DT * 1e9), StateWrap(name, value(v))) for i, v in enumerate(old)]
+ states = [StateWrap(name, value(v)) for v in new]
+ lines.append(f"\n {name} ({len(diffs)} diffs)")
+ if full and not isinstance(diffs[0][2][0], bool):
+ for i in range(0, len(diffs), 10):
+ lines += format_numeric_diffs(diffs[i:i + 10])
+ else:
+ lines += format_diff(diffs, ref, states, name)
+ return lines
+
+
+def diff_report(dumps: Path, base: Path, new: Path, errors: list[str]) -> int:
+ changed, changed_full, new_cars, n_passed = [], [], [], 0
+ for dump in sorted(dumps.glob("*.pkl"), key=lambda p: int(p.stem)):
+ with open(dump, "rb") as f:
+ route, model, mode, _, frames = pickle.load(f)
+ a, b = dict(np.load(base / f"{dump.stem}.npz")), dict(np.load(new / f"{dump.stem}.npz"))
+ if "ok" not in b:
+ errors.append(f"{model} - {route}: this opendbc has no safety mode {mode}")
+ elif "ok" not in a:
+ new_cars.append(f"\nNEW {model} - {route}: master's opendbc has no safety mode {mode}")
+ else:
+ messages, states = message_diffs(frames, a, b), state_diffs(a, b)
+ if messages or states:
+ changed.append([f"\n{model} - {route}"] + messages + states)
+ changed_full.append([f"\n{model} - {route}"] + messages + state_diffs(a, b, full=True))
+ else:
+ n_passed += 1
+
+ from openpilot.common.basedir import BASEDIR
+ from openpilot.selfdrive.test.diff_format import format_report
+
+ def fmt(shown, notes):
+ return format_report("Safety replay diff report", "Replays every car test route through this PR's panda safety and compares the behavior to master.",
+ shown, n_passed, errors, new_cars + notes, n_changed=len(changed))
+
+ (Path(BASEDIR) / "diff_safety_full.txt").write_text(fmt(changed_full, []))
+
+ # the comment shows as many routes as fit, the full report is in the job's artifact
+ notes = []
+ if changed and "GITHUB_RUN_ID" in os.environ:
+ run = f"{os.environ['GITHUB_SERVER_URL']}/{os.environ['GITHUB_REPOSITORY']}/actions/runs/{os.environ['GITHUB_RUN_ID']}"
+ notes.append(f"\nEvery changed frame: diff_safety_full.txt in {run}")
+ n = len(changed)
+ while n and len(fmt(changed[:n] + [[f"\n(... {len(changed) - n} more routes)"]], notes)) > MAX_REPORT:
+ n -= 1
+ shown = changed if n == len(changed) else changed[:n] + [[f"\n(... {len(changed) - n} more routes)"]]
+ report = fmt(shown, notes)
+ print(report)
+ (Path(BASEDIR) / "diff_safety.txt").write_text(report)
+ return int(bool(changed or errors))
+
+
+def main() -> int:
+ # runs with the base's opendbc, so it only uses libsafety
+ if sys.argv[1:2] == ["results"]:
+ dumps, out = Path(sys.argv[2]), Path(sys.argv[3])
+ out.mkdir()
+ from opendbc.safety.tests.libsafety import libsafety_py
+ build = libsafety_py._build_libsafety()
+ with pool() as p:
+ list(p.map(results_route, [(d, out, build) for d in sorted(dumps.glob("*.pkl"))]))
+ return 0
+
+ from openpilot.common.basedir import BASEDIR
+ root = Path(BASEDIR)
+
+ def git(*args, cwd=root):
+ return subprocess.check_output(["git", *args], cwd=cwd, text=True).strip()
+
+ check = sys.argv[1:2] == ["check"]
+ base_ref = (sys.argv[2:] if check else sys.argv[1:]) or ["HEAD^"]
+ base, new = git("rev-parse", f"{base_ref[0]}:opendbc_repo"), git("rev-parse", "HEAD:opendbc_repo")
+ # for a GitHub step's outputs
+ if check:
+ print(f"changed={str(base != new).lower()}")
+ return 0
+ if base == new:
+ print(f"opendbc is unchanged ({new}), nothing to compare")
+ return 0
+ print(f"comparing opendbc {base} -> {new}", flush=True)
+
+ from opendbc.car.tests.routes import routes
+ with tempfile.TemporaryDirectory() as tmp:
+ w, odbc = Path(tmp), root / "opendbc_repo"
+ git("fetch", "-q", "--depth=1", "origin", base, cwd=odbc)
+ git("worktree", "add", "-q", "--detach", str(w / "base"), base, cwd=odbc)
+ try:
+ (w / "routes").mkdir()
+ with pool() as p:
+ errors = [e for e in p.map(dump_route, [(i, r, w / "routes") for i, r in enumerate(routes)]) if e]
+ for name, path in (("base", w / "base"), ("new", odbc)):
+ subprocess.check_call([sys.executable, __file__, "results", w / "routes", w / f"{name}_results"], env={**os.environ, "PYTHONPATH": str(path)})
+ return diff_report(w / "routes", w / "base_results", w / "new_results", errors)
+ finally:
+ git("worktree", "remove", "--force", str(w / "base"), cwd=odbc)
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/openpilot/selfdrive/pandad/tests/safety_replay.py b/openpilot/selfdrive/pandad/tests/safety_replay.py
new file mode 100755
index 00000000000000..ae85fad2326a21
--- /dev/null
+++ b/openpilot/selfdrive/pandad/tests/safety_replay.py
@@ -0,0 +1,251 @@
+#!/usr/bin/env python3
+# checks the panda's safety against libsafety on the car test routes
+import hashlib
+import re
+import multiprocessing
+import os
+import subprocess
+import sys
+import time
+import tomllib
+from collections import Counter
+from concurrent.futures import ProcessPoolExecutor
+from concurrent.futures.process import BrokenProcessPool
+from pathlib import Path
+
+from opendbc.car.tests.routes import routes
+from opendbc.car.tests.test_models import NUM_JOBS, JOB_ID
+from opendbc.safety.tests.libsafety import libsafety_py
+from openpilot.common.basedir import BASEDIR
+from openpilot.common.hardware import HARDWARE
+from openpilot.selfdrive.pandad.tests.diff_safety import BUSES, DT, load, replay
+
+SECONDS_PER_ROUTE = 10
+# seconds until the RX checks are compared
+STEADY = 2
+# frames the station's CAN lanes can start apart
+SKEW = 3
+# master's passing runs go in alfred's first cache dir, which jobs get back from master's last good job
+PASSED = Path(tomllib.loads((Path(BASEDIR) / "alfred.toml").read_text())["cache"][0]) / "safety_replay"
+# a pass is reused for a day, so the rack itself is still checked daily
+PASS_TTL = 24 * 3600
+
+
+def inputs() -> str:
+ # everything the result depends on, except the rack: git's ids of opendbc, panda and this test's dir, and the OS that builds them
+ ids = subprocess.check_output(["git", "rev-parse", "HEAD:opendbc_repo", "HEAD:panda", "HEAD:openpilot/selfdrive/pandad"], cwd=BASEDIR, text=True)
+ return hashlib.sha256(f"{ids} {HARDWARE.get_os_version()} {NUM_JOBS} {JOB_ID}".encode()).hexdigest()[:16]
+
+
+def relay_malfunction_fault() -> int:
+ # the fault bit as the panda firmware defines it
+ sys_h = (Path(BASEDIR) / "panda/board/sys/sys.h").read_text()
+ return 1 << int(re.search(r"#define FAULT_RELAY_MALFUNCTION\s+\(1UL << (\d+)\)", sys_h).group(1))
+
+
+def passed_age(key: str) -> float | None:
+ try:
+ # wall clock, the pass was written on another DUT
+ age = time.time() - float((PASSED / key).read_text()) # noqa: TID251
+ except (OSError, ValueError):
+ return None
+ return age if 0 <= age < PASS_TTL else None
+
+
+def load_window(route):
+ # a route download can drop mid file
+ for attempt in range(3):
+ try:
+ return load(route, SECONDS_PER_ROUTE)
+ except Exception:
+ if attempt == 2:
+ raise
+ time.sleep(10)
+
+
+def without_forwarded_copies(mode, param, frames):
+ # drop the panda's forwarded copies in the log, the panda forwards them again
+ safety = libsafety_py.libsafety
+ assert safety.set_safety_hooks(mode, param) == 0
+ out = []
+ for frame in frames:
+ copies = Counter(frame)
+ drop = Counter()
+ for (addr, dat, bus), n in copies.items():
+ if bus == 0 and (addr, dat, 2) in copies and safety.safety_fwd_hook(0, addr) == 2:
+ drop[(addr, dat, 2)] += min(n, copies[(addr, dat, 2)])
+ elif bus == 2 and (addr, dat, 0) in copies and safety.safety_fwd_hook(2, addr) == 0 and safety.safety_fwd_hook(0, addr) != 2:
+ drop[(addr, dat, 0)] += min(n, copies[(addr, dat, 0)])
+ kept = []
+ for msg in frame:
+ if drop[msg]:
+ drop[msg] -= 1
+ else:
+ kept.append(msg)
+ out.append(kept)
+ return out
+
+
+def expected(mode, param, frames):
+ checks_invalid = []
+
+ def on_tick(i, safety):
+ checks_invalid.append(i * DT >= STEADY and bool(safety.safety_a.state.safety_rx_checks_invalid))
+
+ steps = replay(mode, param, frames, on_tick)
+ rx, fwd, invalid = Counter(), Counter(), 0
+ for frame, (results, _) in zip(frames, steps, strict=True):
+ for (_, _, bus), (valid, destination) in zip(frame, results, strict=True):
+ rx[bus] += 1
+ fwd[bus] += destination >= 0
+ invalid += not valid
+ return {"rx": [rx[b] for b in range(BUSES)], "fwd": [fwd[b] for b in range(BUSES)], "rx_invalid": invalid,
+ "relay_malfunction": bool(libsafety_py.libsafety.get_relay_malfunction()), "rx_checks_invalid": any(checks_invalid),
+ "safety": [mode, param], "faults": 0, "tx_buffer_overflow": 0, "heartbeat_disabled": False, "harness_changed": False}
+
+
+def skewed(frames, bus, k):
+ # this bus's lane starts k frames later
+ out = [[m for m in f if m[2] != bus] for f in [[]] * SKEW + frames + [[]] * SKEW]
+ for i, f in enumerate(frames):
+ out[SKEW + i + k] += [m for m in f if m[2] == bus]
+ return out
+
+
+def bounds(runs):
+ def pick(fn, vals):
+ if isinstance(vals[0], list):
+ return [pick(fn, list(v)) for v in zip(*vals, strict=True)]
+ return fn(vals)
+ return ({k: pick(min, [r[k] for r in runs]) for k in runs[0]}, {k: pick(max, [r[k] for r in runs]) for k in runs[0]})
+
+
+def prepare(route):
+ mode, param, frames = load_window(route)
+ frames = without_forwarded_copies(mode, param, frames)
+ # the UP sends a 10 ms step's frames on different buses in any order, and its lanes start up to SKEW frames apart
+ runs = [expected(mode, param, [sorted(f, key=lambda m: m[2] * order) for f in skewed(frames, 2, k)])
+ for order in (1, -1) for k in range(-SKEW, SKEW + 1)]
+ lo, hi = bounds(runs)
+ return mode, param, frames, lo, hi
+
+
+def new_pool():
+ return ProcessPoolExecutor(1, mp_context=multiprocessing.get_context("spawn"))
+
+
+def within(lo, hi, got):
+ if isinstance(lo, list):
+ return all(within(a, b, g) for a, b, g in zip(lo, hi, got, strict=True))
+ return min(lo, hi) <= got <= max(lo, hi)
+
+
+def played(p, up, mode, param, frames):
+ # first, it resets the heartbeat timeout that turns power save back on
+ p.set_safety_mode(mode, param)
+ # the panda went into power save while the route loaded without a heartbeat
+ p.set_power_save(0)
+ # can_health is per CAN controller, a flipped harness swaps buses 0 and 2
+ cans = [2, 1, 0] if p.health()["car_harness_status"] == p.HARNESS_STATUS_FLIPPED else [0, 1, 2]
+ hs0, h0 = p.health(), [p.can_health(c) for c in cans]
+ fd = {bus for frame in frames for _, dat, bus in frame if len(dat) > 8}
+ # the panda forwards CAN FD frames between buses 0 and 2
+ if fd & {0, 2}:
+ fd |= {0, 2}
+ for b in fd:
+ up.set_can_data_speed_kbps(b, h0[b]["can_data_speed"])
+ # pad the end so the UP doesn't loop the route
+ up.can_replay(frames + [[]] * int(60 / DT))
+ start, checks_invalid = time.monotonic(), False
+ while time.monotonic() < start + len(frames) * DT + 1:
+ p.send_heartbeat(engaged=False)
+ # the panda updates its RX checks once a second
+ if STEADY + 1 <= time.monotonic() - start <= len(frames) * DT:
+ checks_invalid |= p.health()["safety_rx_checks_invalid"]
+ time.sleep(0.2)
+ h, h1 = p.health(), [p.can_health(c) for c in cans]
+ relay = relay_malfunction_fault()
+ up.reset()
+ up.set_ignition(True)
+ return {"rx": [h1[b]["total_rx_cnt"] - h0[b]["total_rx_cnt"] for b in range(BUSES)],
+ "fwd": [h1[b]["total_fwd_cnt"] - h0[b]["total_fwd_cnt"] for b in range(BUSES)],
+ "rx_invalid": h["safety_rx_invalid"], "relay_malfunction": bool(h["faults"] & relay), "rx_checks_invalid": checks_invalid,
+ "safety": [h["safety_mode"], h["safety_param"]],
+ "faults": h["faults"] & ~hs0["faults"] & ~relay,
+ # TX only, RX overflows without pandad
+ "tx_buffer_overflow": h["tx_buffer_overflow"] - hs0["tx_buffer_overflow"],
+ "heartbeat_disabled": h["heartbeat_disabled"], "harness_changed": h["car_harness_status"] != hs0["car_harness_status"],
+ # only printed, to debug a failure
+ "can_errors": {k: [h1[b][k] - h0[b][k] for b in range(BUSES)]
+ for k in ("total_error_cnt", "total_tx_lost_cnt", "bus_off_cnt", "can_core_reset_count")}}
+
+
+def main():
+ from panda import Panda
+ from ultimate_provisioning import UP
+ from openpilot.selfdrive.pandad.pandad import flash_panda
+
+ key = inputs()
+ age = passed_age(key)
+ if age is None:
+ print(f"inputs {key} have no master pass in the last {PASS_TTL // 3600} h, replaying", flush=True)
+ elif os.environ.get("SAFETY_REPLAY_FORCE") == "1":
+ print(f"inputs {key} passed on master {age / 3600:.1f} h ago, replaying anyway (SAFETY_REPLAY_FORCE)", flush=True)
+ else:
+ print(f"skipped: master passed these inputs ({key}) {age / 3600:.1f} h ago, same firmware, opendbc, panda library, test and OS")
+ return 0
+
+ flash_panda()
+ os.environ["SAFETY_UBSAN_TRAP"] = "1"
+ p, up = Panda(), UP.remote()
+ # the UP starts with the harness lines floating
+ up.reset()
+ up.set_ignition(True)
+
+ names = sys.argv[1:]
+ todo = [r for i, r in enumerate(routes) if i % NUM_JOBS == JOB_ID and (not names or r.route in names)]
+ failed = []
+ # prepares the next route while this one replays
+ pool = new_pool()
+ loading = pool.submit(prepare, todo[0]) if todo else None
+ for n, route in enumerate(todo):
+ st = time.monotonic()
+ try:
+ prepared = loading.result()
+ except BrokenProcessPool as e:
+ # a crash in libsafety, like a UBSan trap
+ prepared = e
+ pool = new_pool()
+ except Exception as e:
+ prepared = e
+ if n + 1 < len(todo):
+ loading = pool.submit(prepare, todo[n + 1])
+ if isinstance(prepared, Exception):
+ print(f"{route.route} FAIL, can't prepare: {type(prepared).__name__}: {prepared!r}", flush=True)
+ failed.append(route.route)
+ continue
+ mode, param, frames, lo, hi = prepared
+ got = played(p, up, mode, param, frames)
+ # forwarding stops at a relay malfunction, exactly when depends on the tick
+ if lo["relay_malfunction"] or hi["relay_malfunction"]:
+ for d in (lo, hi, got):
+ del d["fwd"]
+ ok = all(within(lo[k], hi[k], got[k]) for k in lo)
+ exp = lo if lo == hi else {"low": lo, "high": hi}
+ dropped = [e - g for e, g in zip(lo["rx"], got["rx"], strict=True)]
+ print(f"{route.route} {route.car_model} mode {mode} {'ok' if ok else 'FAIL'} {time.monotonic() - st:.0f}s expected {exp} panda {got}" +
+ (f" UP dropped {dropped}" if any(dropped) else ""), flush=True)
+ if not ok:
+ failed.append(route.route)
+
+ pool.shutdown()
+ print(f"{len(failed)} failed: {failed}")
+ if failed or names:
+ return int(bool(failed))
+ PASSED.mkdir(parents=True, exist_ok=True)
+ (PASSED / key).write_text(str(time.time())) # noqa: TID251
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/openpilot/selfdrive/test/diff_format.py b/openpilot/selfdrive/test/diff_format.py
new file mode 100644
index 00000000000000..806867fed6cf1e
--- /dev/null
+++ b/openpilot/selfdrive/test/diff_format.py
@@ -0,0 +1,20 @@
+# the layout of the diff report comments, light to import: CI jobs without a build use it
+
+
+def format_report(title: str, description: str, changes: list[list[str]], n_passed: int, errors: list[str], notes: list[str] | None = None,
+ n_changed: int | None = None) -> str:
+ icon = "⚠️" if changes else "✅"
+ lines = [
+ f"## {title}",
+ description,
+ "Please review any changes carefully to ensure they are expected.\n",
+ f"{icon} {len(changes) if n_changed is None else n_changed} changed, {n_passed} passed, {len(errors)} errors",
+ ]
+ lines += [f"\nERROR {e}" for e in errors]
+ lines += notes or []
+ if changes:
+ lines.append("Show changes
\n\n```")
+ for change in changes:
+ lines += change
+ lines.append("```\n ")
+ return "\n".join(lines)
diff --git a/openpilot/selfdrive/test/process_replay/diff_report.py b/openpilot/selfdrive/test/process_replay/diff_report.py
index c6c1f01abd3c36..f91b61ea541d84 100644
--- a/openpilot/selfdrive/test/process_replay/diff_report.py
+++ b/openpilot/selfdrive/test/process_replay/diff_report.py
@@ -1,6 +1,7 @@
import os
from collections import defaultdict
from opendbc.car.tests.car_diff import format_diff, format_numeric_diffs
+from openpilot.selfdrive.test.diff_format import format_report
from openpilot.selfdrive.test.process_replay.compare_logs import compare_logs
from openpilot.selfdrive.test.process_replay.process_replay import PROC_REPLAY_DIR
@@ -66,28 +67,18 @@ def diff_report(replay_diffs, segments) -> None:
else:
with_diffs.append((plat, seg, proc, data))
- icon = "⚠️" if with_diffs else "✅"
- lines = [
- "## Process replay diff report",
- "Replays driving segments through this PR and compares the behavior to master.",
- "Please review any changes carefully to ensure they are expected.\n",
- f"{icon} {len(with_diffs)} changed, {n_passed} passed, {len(errors)} errors",
- ]
-
- for plat, seg, proc, err in errors:
- lines.append(f"\nERROR {plat} - {seg} [{proc}]: {err}")
-
- if with_diffs:
- lines.append("Show changes
\n\n```")
- for plat, seg, proc, (diffs, ref, new) in with_diffs:
- lines.append(f"\n{plat} - {seg} [{proc}]")
- by_field = defaultdict(list)
- for d in diffs:
- by_field[d[0]].append(d)
- for field, fd in sorted(by_field.items()):
- lines.append(f"\n {field} ({len(fd)} diffs)")
- lines.extend(diff_format(fd, ref, new, field))
- lines.append("```\n ")
+ changes = []
+ for plat, seg, proc, (diffs, ref, new) in with_diffs:
+ lines = [f"\n{plat} - {seg} [{proc}]"]
+ by_field = defaultdict(list)
+ for d in diffs:
+ by_field[d[0]].append(d)
+ for field, fd in sorted(by_field.items()):
+ lines.append(f"\n {field} ({len(fd)} diffs)")
+ lines.extend(diff_format(fd, ref, new, field))
+ changes.append(lines)
+ report = format_report("Process replay diff report", "Replays driving segments through this PR and compares the behavior to master.",
+ changes, n_passed, [f"{plat} - {seg} [{proc}]: {err}" for plat, seg, proc, err in errors])
with open(os.path.join(PROC_REPLAY_DIR, "diff_report.txt"), "w") as f:
- f.write("\n".join(lines))
+ f.write(report)
diff --git a/panda b/panda
index bf14f329132082..8ae88ad1d93096 160000
--- a/panda
+++ b/panda
@@ -1 +1 @@
-Subproject commit bf14f329132082db7559ddca33598be7c6272528
+Subproject commit 8ae88ad1d93096c64e3e9fc143853ad7bd9ddac9