diff --git a/.github/workflows/diff_report.yaml b/.github/workflows/diff_report.yaml index 202bc0ec7934af..5c6013286c2387 100644 --- a/.github/workflows/diff_report.yaml +++ b/.github/workflows/diff_report.yaml @@ -48,11 +48,40 @@ jobs: run-id: ${{ steps.tests-run.outputs.run-id }} name: diff_report_${{ github.event.number }} path: . + - name: Wait for safety replay + if: steps.wait.outcome == 'success' + id: wait-safety + continue-on-error: true + uses: lewagon/wait-on-check-action@1d57e2c51a58d812d2765e036a028b6bdb5a6154 + with: + ref: ${{ github.event.pull_request.head.sha }} + check-name: safety replay + repo-token: ${{ secrets.GITHUB_TOKEN }} + allowed-conclusions: success,failure + wait-interval: 20 + - name: Download safety diff + if: steps.wait-safety.outcome == 'success' + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + repository: ${{ github.repository }} + run-id: ${{ steps.tests-run.outputs.run-id }} + name: diff_safety_${{ github.event.number }} + path: . + # one comment for both reports, the safety one only when this PR changes opendbc + - name: Combine reports + if: steps.wait.outcome == 'success' + run: | + for f in diff_report.txt diff_safety.txt; do + if [ -f "$f" ]; then cat "$f"; printf '\n\n'; fi + done > comment.md + [ -s comment.md ] || rm -f comment.md - name: Comment on PR - if: steps.wait.outcome == 'success' && hashFiles('diff_report.txt') != '' + if: steps.wait.outcome == 'success' && hashFiles('comment.md') != '' uses: thollander/actions-comment-pull-request@24bffb9b452ba05a4f3f77933840a6a841d1b32b with: - file-path: diff_report.txt + file-path: comment.md comment-tag: diff_report pr-number: ${{ github.event.number }} github-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 4777bc9c427c30..6e0ef9b5b144c7 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -174,6 +174,46 @@ jobs: max_attempts: 3 command: cd ${{ github.workspace }}/ci-artifacts && git push origin process-replay --force + safety_replay: + name: safety replay + runs-on: ${{ + (github.repository == 'commaai/openpilot') && + ((github.event_name != 'pull_request') || + (github.event.pull_request.head.repo.full_name == 'commaai/openpilot')) + && fromJSON('["namespace-profile-amd64-8x16"]') + || fromJSON('["ubuntu-24.04"]') }} + env: + COMMA_CACHE: /tmp/comma_download_cache + steps: + - uses: actions/checkout@v7 + with: + # the parent is master, or the previous master commit on a push + fetch-depth: 2 + - run: ./tools/op.sh setup + - name: Check for an opendbc change + id: opendbc + run: openpilot/selfdrive/pandad/tests/diff_safety.py check >> "$GITHUB_OUTPUT" + - name: Cache the test routes + if: steps.opendbc.outputs.changed == 'true' + uses: actions/cache@v6 + with: + path: /tmp/comma_download_cache + key: safety-replay-${{ hashFiles('opendbc_repo/opendbc/car/tests/routes.py') }} + restore-keys: safety-replay- + - name: Compare safety with the previous opendbc + if: steps.opendbc.outputs.changed == 'true' + timeout-minutes: 20 + run: openpilot/selfdrive/pandad/tests/diff_safety.py + - name: Upload diff report + if: always() && github.event_name == 'pull_request' + uses: actions/upload-artifact@v7 + with: + name: diff_safety_${{ github.event.number }} + path: | + diff_safety.txt + diff_safety_full.txt + if-no-files-found: ignore + simulator_driving: name: simulator driving runs-on: ${{ diff --git a/alfred.toml b/alfred.toml index 9ba6ebeda2cd9f..3b96e10a08c730 100644 --- a/alfred.toml +++ b/alfred.toml @@ -23,6 +23,10 @@ for p in msgq_repo/msgq opendbc_repo/opendbc rednose_repo/rednose teleoprtc_repo tools/op.sh build """ +[steps.safety-replay] +name = "Safety replay" +run = "./openpilot/selfdrive/pandad/tests/safety_replay.py" + [steps.flash-chestnut] name = "Flash Chestnut" run = "sudo python3 openpilot/system/hardware/chestnut/flash.py" @@ -117,6 +121,55 @@ uses = "build" name = "Sensord" run = "./openpilot/system/sensord/tests/test_sensord.py" +# each job replays an eighth of the routes, unless master passed the same inputs in the last day +[jobs.safety_0] +runs-on = "up-can" +timeout-minutes = 25 +env = { NUM_JOBS = "8", JOB_ID = "0" } +steps = [{ uses = "build" }, { uses = "safety-replay" }] + +[jobs.safety_1] +runs-on = "up-can" +timeout-minutes = 25 +env = { NUM_JOBS = "8", JOB_ID = "1" } +steps = [{ uses = "build" }, { uses = "safety-replay" }] + +[jobs.safety_2] +runs-on = "up-can" +timeout-minutes = 25 +env = { NUM_JOBS = "8", JOB_ID = "2" } +steps = [{ uses = "build" }, { uses = "safety-replay" }] + +[jobs.safety_3] +runs-on = "up-can" +timeout-minutes = 25 +env = { NUM_JOBS = "8", JOB_ID = "3" } +steps = [{ uses = "build" }, { uses = "safety-replay" }] + +[jobs.safety_4] +runs-on = "up-can" +timeout-minutes = 25 +env = { NUM_JOBS = "8", JOB_ID = "4" } +steps = [{ uses = "build" }, { uses = "safety-replay" }] + +[jobs.safety_5] +runs-on = "up-can" +timeout-minutes = 25 +env = { NUM_JOBS = "8", JOB_ID = "5" } +steps = [{ uses = "build" }, { uses = "safety-replay" }] + +[jobs.safety_6] +runs-on = "up-can" +timeout-minutes = 25 +env = { NUM_JOBS = "8", JOB_ID = "6" } +steps = [{ uses = "build" }, { uses = "safety-replay" }] + +[jobs.safety_7] +runs-on = "up-can" +timeout-minutes = 25 +env = { NUM_JOBS = "8", JOB_ID = "7" } +steps = [{ uses = "build" }, { uses = "safety-replay" }] + # a comma four with a Chestnut on its aux port, which runs the big model [jobs.chestnut] runs-on = "chestnut" diff --git a/opendbc_repo b/opendbc_repo index 27fa1c5d8eed32..ec18ed089a32e4 160000 --- a/opendbc_repo +++ b/opendbc_repo @@ -1 +1 @@ -Subproject commit 27fa1c5d8eed3238a0a78c52ff5abb6cdc807bd2 +Subproject commit ec18ed089a32e4bdc7e259e1f2d0725baf73a48a diff --git a/openpilot/selfdrive/pandad/tests/diff_safety.py b/openpilot/selfdrive/pandad/tests/diff_safety.py new file mode 100755 index 00000000000000..1d8b1191c8bb80 --- /dev/null +++ b/openpilot/selfdrive/pandad/tests/diff_safety.py @@ -0,0 +1,298 @@ +#!/usr/bin/env python3 +# replays the car test routes through BASE's and this opendbc's safety, reports every changed RX and TX result and safety state value +# diff_safety.py [BASE] BASE defaults to HEAD^, which is master on a PR's merge commit +# diff_safety.py check [BASE] prints changed=true when opendbc differs from BASE's +import multiprocessing +import os +import pickle +import shutil +import subprocess +import sys +import tempfile +from collections import Counter +from concurrent.futures import ProcessPoolExecutor +from pathlib import Path +from typing import Any + +import numpy as np + +from opendbc.car import DT_CTRL as DT + +# GitHub comments hold 65536 characters, and this shares one with the process replay report +MAX_REPORT = 50000 +# the panda's CAN buses, a log's can src is bus + 4 * panda +BUSES = 3 +# the panda marks sent CAN with src + 128 and blocked with src + 192 +TX = 128 + + +def load(route, seconds: float | None = None, tx: bool = False): + from opendbc.car.car_helpers import interfaces + from opendbc.car.tests.test_models import TestCarModelBase + + R = type("R", (TestCarModelBase,), {"test_route": route, "platform": route.car_model}) + car_fw, can_msgs, alpha_long = R.get_testing_data(route) + CP = interfaces[R.platform].get_params(R.platform, R.fingerprint, car_fw, alpha_long, False, docs=False) + cfg = CP.safetyConfigs[-1] + + t0 = can_msgs[0][0] + steps = int((can_msgs[-1][0] - t0) / 1e9 / DT) + 1 if seconds is None else int(seconds / DT) + frames = [[] for _ in range(steps)] + for t, msgs in can_msgs: + i = int((t - t0) / 1e9 / DT) + if i >= len(frames): + break + for m in msgs: + if m.src < TX and m.src % 4 < BUSES: + frames[i].append((m.address, bytes(m.dat), m.src % 4)) + # sent and blocked TX, so a check that allows more shows up too + elif tx and m.src >= TX and m.src % 4 < BUSES: + frames[i].append((m.address, bytes(m.dat), TX + m.src % 4)) + return cfg.safetyModel.raw, cfg.safetyParam, frames + + +def replay(mode: int, param: int, frames, on_tick=None): + # the result of each message and the whole safety state (safety_a) after each 10 ms step + from opendbc.safety.tests.libsafety import libsafety_py + safety = libsafety_py.libsafety + assert safety.set_safety_hooks(mode, param) == 0 + safety.set_alternative_experience(0) + # older opendbc names it safety_tick_current_safety_config + tick = safety.safety_tick if hasattr(safety, "safety_tick") else safety.safety_tick_current_safety_config + state = libsafety_py.ffi.buffer(libsafety_py.ffi.addressof(safety, "safety_a")) + + steps = [] + for i, frame in enumerate(frames): + safety.set_timer(int(i * DT * 1e6)) + if i % int(1 / DT) == 0: + tick() + if on_tick is not None: + on_tick(i, safety) + results = [] + for addr, dat, bus in frame: + if bus >= TX: + results.append(bool(safety.safety_tx_hook(libsafety_py.make_CANPacket(addr, bus - TX, dat)))) + else: + fwd = safety.safety_fwd_hook(bus, addr) + results.append((bool(safety.safety_rx_hook(libsafety_py.make_CANPacket(addr, bus, dat))), fwd)) + steps.append((results, state[:])) + return steps + + +def dump_route(args): + i, route, out = args + try: + with open(out / f"{i}.pkl", "wb") as f: + pickle.dump((route.route, str(route.car_model), *load(route, tx=True)), f) + except Exception as e: + return f"{route.car_model} - {route.route}: can't load, {e!r}" + return None + + +def state_layout(ffi): + # every field of the safety state with its offset and type, pointers left out since they differ between processes + fields = [] + + def walk(t, path, offset): + if t.kind == "struct": + for name, f in t.fields: + walk(f.type, f"{path}.{name}" if path else name, offset + f.offset) + elif t.kind == "array": + for i in range(t.length): + walk(t.item, f"{path}[{i}]", offset + i * ffi.sizeof(t.item)) + elif t.kind in ("primitive", "enum"): + size = ffi.sizeof(t) + if t.cname == "_Bool": + dtype = "?" + else: + # cffi only converts float types with float() + try: + float(ffi.cast(t, 0)) + dtype = f"f{size}" + except TypeError: + dtype = f"i{size}" if int(ffi.cast(t, -1)) < 0 else f"u{size}" + fields.append((path.removeprefix("state."), offset, size, dtype)) + + walk(ffi.typeof("SafetyCopy"), "", 0) + return fields + + +def results_route(args): + # every message's result (RX valid or TX allowed), where each RX is forwarded, and each safety state field after each step + path, out, build = args + with open(path, "rb") as f: + route, model, mode, param, frames = pickle.load(f) + from opendbc.safety.tests.libsafety import libsafety_py + # a copy of the build is a fresh library, no safety state carries over from another route + fd, copy = tempfile.mkstemp(suffix=".so") + os.close(fd) + shutil.copy(build, copy) + libsafety_py.load(copy) + os.unlink(copy) + if libsafety_py.libsafety.set_safety_hooks(mode, param) != 0: + np.savez(out / path.with_suffix(".npz").name, mode=mode) + return + steps = replay(mode, param, frames) + results = [r for step_results, _ in steps for r in step_results] + raw = np.frombuffer(b"".join(state for _, state in steps), dtype=np.uint8).reshape(len(steps), -1) + arrays: dict[str, Any] = {f"state:{name}": raw[:, offset:offset + size].copy().view(dtype)[:, 0] + for name, offset, size, dtype in state_layout(libsafety_py.ffi)} + arrays["ok"] = np.array([r[0] if isinstance(r, tuple) else r for r in results], dtype=bool) + arrays["fwd"] = np.array([r[1] if isinstance(r, tuple) else -2 for r in results], dtype=np.int16) + np.savez_compressed(out / path.with_suffix(".npz").name, **arrays) + + +def pool(): + return ProcessPoolExecutor(os.cpu_count(), mp_context=multiprocessing.get_context("spawn")) + + +def value(v): + return bool(v) if isinstance(v, np.bool_) else v.item() + + +def message_diffs(frames, a, b) -> list[str]: + # RX valid, RX forward and TX allowed, grouped by message and change + events = [(i, addr, bus) for i, frame in enumerate(frames) for addr, _, bus in frame] + changes = Counter() + first = {} + for k in np.flatnonzero((a["ok"] != b["ok"]) | (a["fwd"] != b["fwd"])): + i, addr, bus = events[k] + if bus >= TX: + key = (f"TX 0x{addr:x} bus {bus - TX} allowed", bool(a["ok"][k]), bool(b["ok"][k])) + elif a["ok"][k] != b["ok"][k]: + key = (f"RX 0x{addr:x} bus {bus} valid", bool(a["ok"][k]), bool(b["ok"][k])) + else: + key = (f"RX 0x{addr:x} bus {bus} forward", int(a["fwd"][k]), int(b["fwd"][k])) + changes[key] += 1 + first.setdefault(key, i) + return [f"\n {what}: {old} -> {new} ({n} msgs, from frame {first[(what, old, new)]})" for (what, old, new), n in sorted(changes.items())] + + +class StateWrap: + """Adapter so car_diff reads one safety state field, dotted names are nested like in a struct""" + def __init__(self, name: str, v): + *parents, leaf = name.split(".") + self._d = d = {} + for p in parents: + d = d.setdefault(p, {}) + d[leaf] = v + def to_dict(self) -> dict: + return self._d + + +def state_diffs(a, b, full: bool = False) -> list[str]: + # full: every changed frame instead of the first ones, for the report in the artifact + from opendbc.car.tests.car_diff import format_diff, format_numeric_diffs + lines = [] + for key in sorted(a.keys() | b.keys()): + if not key.startswith("state:"): + continue + name = key.removeprefix("state:") + if key not in a or key not in b: + lines.append(f"\n {name}: only in {'master' if key in a else 'this PR'}") + continue + old, new = a[key], b[key] + frames = np.flatnonzero(old != new) + if not len(frames): + continue + diffs = [(name, int(i), (value(old[i]), value(new[i])), int(i * DT * 1e9)) for i in frames] + ref = [(int(i * DT * 1e9), StateWrap(name, value(v))) for i, v in enumerate(old)] + states = [StateWrap(name, value(v)) for v in new] + lines.append(f"\n {name} ({len(diffs)} diffs)") + if full and not isinstance(diffs[0][2][0], bool): + for i in range(0, len(diffs), 10): + lines += format_numeric_diffs(diffs[i:i + 10]) + else: + lines += format_diff(diffs, ref, states, name) + return lines + + +def diff_report(dumps: Path, base: Path, new: Path, errors: list[str]) -> int: + changed, changed_full, new_cars, n_passed = [], [], [], 0 + for dump in sorted(dumps.glob("*.pkl"), key=lambda p: int(p.stem)): + with open(dump, "rb") as f: + route, model, mode, _, frames = pickle.load(f) + a, b = dict(np.load(base / f"{dump.stem}.npz")), dict(np.load(new / f"{dump.stem}.npz")) + if "ok" not in b: + errors.append(f"{model} - {route}: this opendbc has no safety mode {mode}") + elif "ok" not in a: + new_cars.append(f"\nNEW {model} - {route}: master's opendbc has no safety mode {mode}") + else: + messages, states = message_diffs(frames, a, b), state_diffs(a, b) + if messages or states: + changed.append([f"\n{model} - {route}"] + messages + states) + changed_full.append([f"\n{model} - {route}"] + messages + state_diffs(a, b, full=True)) + else: + n_passed += 1 + + from openpilot.common.basedir import BASEDIR + from openpilot.selfdrive.test.diff_format import format_report + + def fmt(shown, notes): + return format_report("Safety replay diff report", "Replays every car test route through this PR's panda safety and compares the behavior to master.", + shown, n_passed, errors, new_cars + notes, n_changed=len(changed)) + + (Path(BASEDIR) / "diff_safety_full.txt").write_text(fmt(changed_full, [])) + + # the comment shows as many routes as fit, the full report is in the job's artifact + notes = [] + if changed and "GITHUB_RUN_ID" in os.environ: + run = f"{os.environ['GITHUB_SERVER_URL']}/{os.environ['GITHUB_REPOSITORY']}/actions/runs/{os.environ['GITHUB_RUN_ID']}" + notes.append(f"\nEvery changed frame: diff_safety_full.txt in {run}") + n = len(changed) + while n and len(fmt(changed[:n] + [[f"\n(... {len(changed) - n} more routes)"]], notes)) > MAX_REPORT: + n -= 1 + shown = changed if n == len(changed) else changed[:n] + [[f"\n(... {len(changed) - n} more routes)"]] + report = fmt(shown, notes) + print(report) + (Path(BASEDIR) / "diff_safety.txt").write_text(report) + return int(bool(changed or errors)) + + +def main() -> int: + # runs with the base's opendbc, so it only uses libsafety + if sys.argv[1:2] == ["results"]: + dumps, out = Path(sys.argv[2]), Path(sys.argv[3]) + out.mkdir() + from opendbc.safety.tests.libsafety import libsafety_py + build = libsafety_py._build_libsafety() + with pool() as p: + list(p.map(results_route, [(d, out, build) for d in sorted(dumps.glob("*.pkl"))])) + return 0 + + from openpilot.common.basedir import BASEDIR + root = Path(BASEDIR) + + def git(*args, cwd=root): + return subprocess.check_output(["git", *args], cwd=cwd, text=True).strip() + + check = sys.argv[1:2] == ["check"] + base_ref = (sys.argv[2:] if check else sys.argv[1:]) or ["HEAD^"] + base, new = git("rev-parse", f"{base_ref[0]}:opendbc_repo"), git("rev-parse", "HEAD:opendbc_repo") + # for a GitHub step's outputs + if check: + print(f"changed={str(base != new).lower()}") + return 0 + if base == new: + print(f"opendbc is unchanged ({new}), nothing to compare") + return 0 + print(f"comparing opendbc {base} -> {new}", flush=True) + + from opendbc.car.tests.routes import routes + with tempfile.TemporaryDirectory() as tmp: + w, odbc = Path(tmp), root / "opendbc_repo" + git("fetch", "-q", "--depth=1", "origin", base, cwd=odbc) + git("worktree", "add", "-q", "--detach", str(w / "base"), base, cwd=odbc) + try: + (w / "routes").mkdir() + with pool() as p: + errors = [e for e in p.map(dump_route, [(i, r, w / "routes") for i, r in enumerate(routes)]) if e] + for name, path in (("base", w / "base"), ("new", odbc)): + subprocess.check_call([sys.executable, __file__, "results", w / "routes", w / f"{name}_results"], env={**os.environ, "PYTHONPATH": str(path)}) + return diff_report(w / "routes", w / "base_results", w / "new_results", errors) + finally: + git("worktree", "remove", "--force", str(w / "base"), cwd=odbc) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/openpilot/selfdrive/pandad/tests/safety_replay.py b/openpilot/selfdrive/pandad/tests/safety_replay.py new file mode 100755 index 00000000000000..ae85fad2326a21 --- /dev/null +++ b/openpilot/selfdrive/pandad/tests/safety_replay.py @@ -0,0 +1,251 @@ +#!/usr/bin/env python3 +# checks the panda's safety against libsafety on the car test routes +import hashlib +import re +import multiprocessing +import os +import subprocess +import sys +import time +import tomllib +from collections import Counter +from concurrent.futures import ProcessPoolExecutor +from concurrent.futures.process import BrokenProcessPool +from pathlib import Path + +from opendbc.car.tests.routes import routes +from opendbc.car.tests.test_models import NUM_JOBS, JOB_ID +from opendbc.safety.tests.libsafety import libsafety_py +from openpilot.common.basedir import BASEDIR +from openpilot.common.hardware import HARDWARE +from openpilot.selfdrive.pandad.tests.diff_safety import BUSES, DT, load, replay + +SECONDS_PER_ROUTE = 10 +# seconds until the RX checks are compared +STEADY = 2 +# frames the station's CAN lanes can start apart +SKEW = 3 +# master's passing runs go in alfred's first cache dir, which jobs get back from master's last good job +PASSED = Path(tomllib.loads((Path(BASEDIR) / "alfred.toml").read_text())["cache"][0]) / "safety_replay" +# a pass is reused for a day, so the rack itself is still checked daily +PASS_TTL = 24 * 3600 + + +def inputs() -> str: + # everything the result depends on, except the rack: git's ids of opendbc, panda and this test's dir, and the OS that builds them + ids = subprocess.check_output(["git", "rev-parse", "HEAD:opendbc_repo", "HEAD:panda", "HEAD:openpilot/selfdrive/pandad"], cwd=BASEDIR, text=True) + return hashlib.sha256(f"{ids} {HARDWARE.get_os_version()} {NUM_JOBS} {JOB_ID}".encode()).hexdigest()[:16] + + +def relay_malfunction_fault() -> int: + # the fault bit as the panda firmware defines it + sys_h = (Path(BASEDIR) / "panda/board/sys/sys.h").read_text() + return 1 << int(re.search(r"#define FAULT_RELAY_MALFUNCTION\s+\(1UL << (\d+)\)", sys_h).group(1)) + + +def passed_age(key: str) -> float | None: + try: + # wall clock, the pass was written on another DUT + age = time.time() - float((PASSED / key).read_text()) # noqa: TID251 + except (OSError, ValueError): + return None + return age if 0 <= age < PASS_TTL else None + + +def load_window(route): + # a route download can drop mid file + for attempt in range(3): + try: + return load(route, SECONDS_PER_ROUTE) + except Exception: + if attempt == 2: + raise + time.sleep(10) + + +def without_forwarded_copies(mode, param, frames): + # drop the panda's forwarded copies in the log, the panda forwards them again + safety = libsafety_py.libsafety + assert safety.set_safety_hooks(mode, param) == 0 + out = [] + for frame in frames: + copies = Counter(frame) + drop = Counter() + for (addr, dat, bus), n in copies.items(): + if bus == 0 and (addr, dat, 2) in copies and safety.safety_fwd_hook(0, addr) == 2: + drop[(addr, dat, 2)] += min(n, copies[(addr, dat, 2)]) + elif bus == 2 and (addr, dat, 0) in copies and safety.safety_fwd_hook(2, addr) == 0 and safety.safety_fwd_hook(0, addr) != 2: + drop[(addr, dat, 0)] += min(n, copies[(addr, dat, 0)]) + kept = [] + for msg in frame: + if drop[msg]: + drop[msg] -= 1 + else: + kept.append(msg) + out.append(kept) + return out + + +def expected(mode, param, frames): + checks_invalid = [] + + def on_tick(i, safety): + checks_invalid.append(i * DT >= STEADY and bool(safety.safety_a.state.safety_rx_checks_invalid)) + + steps = replay(mode, param, frames, on_tick) + rx, fwd, invalid = Counter(), Counter(), 0 + for frame, (results, _) in zip(frames, steps, strict=True): + for (_, _, bus), (valid, destination) in zip(frame, results, strict=True): + rx[bus] += 1 + fwd[bus] += destination >= 0 + invalid += not valid + return {"rx": [rx[b] for b in range(BUSES)], "fwd": [fwd[b] for b in range(BUSES)], "rx_invalid": invalid, + "relay_malfunction": bool(libsafety_py.libsafety.get_relay_malfunction()), "rx_checks_invalid": any(checks_invalid), + "safety": [mode, param], "faults": 0, "tx_buffer_overflow": 0, "heartbeat_disabled": False, "harness_changed": False} + + +def skewed(frames, bus, k): + # this bus's lane starts k frames later + out = [[m for m in f if m[2] != bus] for f in [[]] * SKEW + frames + [[]] * SKEW] + for i, f in enumerate(frames): + out[SKEW + i + k] += [m for m in f if m[2] == bus] + return out + + +def bounds(runs): + def pick(fn, vals): + if isinstance(vals[0], list): + return [pick(fn, list(v)) for v in zip(*vals, strict=True)] + return fn(vals) + return ({k: pick(min, [r[k] for r in runs]) for k in runs[0]}, {k: pick(max, [r[k] for r in runs]) for k in runs[0]}) + + +def prepare(route): + mode, param, frames = load_window(route) + frames = without_forwarded_copies(mode, param, frames) + # the UP sends a 10 ms step's frames on different buses in any order, and its lanes start up to SKEW frames apart + runs = [expected(mode, param, [sorted(f, key=lambda m: m[2] * order) for f in skewed(frames, 2, k)]) + for order in (1, -1) for k in range(-SKEW, SKEW + 1)] + lo, hi = bounds(runs) + return mode, param, frames, lo, hi + + +def new_pool(): + return ProcessPoolExecutor(1, mp_context=multiprocessing.get_context("spawn")) + + +def within(lo, hi, got): + if isinstance(lo, list): + return all(within(a, b, g) for a, b, g in zip(lo, hi, got, strict=True)) + return min(lo, hi) <= got <= max(lo, hi) + + +def played(p, up, mode, param, frames): + # first, it resets the heartbeat timeout that turns power save back on + p.set_safety_mode(mode, param) + # the panda went into power save while the route loaded without a heartbeat + p.set_power_save(0) + # can_health is per CAN controller, a flipped harness swaps buses 0 and 2 + cans = [2, 1, 0] if p.health()["car_harness_status"] == p.HARNESS_STATUS_FLIPPED else [0, 1, 2] + hs0, h0 = p.health(), [p.can_health(c) for c in cans] + fd = {bus for frame in frames for _, dat, bus in frame if len(dat) > 8} + # the panda forwards CAN FD frames between buses 0 and 2 + if fd & {0, 2}: + fd |= {0, 2} + for b in fd: + up.set_can_data_speed_kbps(b, h0[b]["can_data_speed"]) + # pad the end so the UP doesn't loop the route + up.can_replay(frames + [[]] * int(60 / DT)) + start, checks_invalid = time.monotonic(), False + while time.monotonic() < start + len(frames) * DT + 1: + p.send_heartbeat(engaged=False) + # the panda updates its RX checks once a second + if STEADY + 1 <= time.monotonic() - start <= len(frames) * DT: + checks_invalid |= p.health()["safety_rx_checks_invalid"] + time.sleep(0.2) + h, h1 = p.health(), [p.can_health(c) for c in cans] + relay = relay_malfunction_fault() + up.reset() + up.set_ignition(True) + return {"rx": [h1[b]["total_rx_cnt"] - h0[b]["total_rx_cnt"] for b in range(BUSES)], + "fwd": [h1[b]["total_fwd_cnt"] - h0[b]["total_fwd_cnt"] for b in range(BUSES)], + "rx_invalid": h["safety_rx_invalid"], "relay_malfunction": bool(h["faults"] & relay), "rx_checks_invalid": checks_invalid, + "safety": [h["safety_mode"], h["safety_param"]], + "faults": h["faults"] & ~hs0["faults"] & ~relay, + # TX only, RX overflows without pandad + "tx_buffer_overflow": h["tx_buffer_overflow"] - hs0["tx_buffer_overflow"], + "heartbeat_disabled": h["heartbeat_disabled"], "harness_changed": h["car_harness_status"] != hs0["car_harness_status"], + # only printed, to debug a failure + "can_errors": {k: [h1[b][k] - h0[b][k] for b in range(BUSES)] + for k in ("total_error_cnt", "total_tx_lost_cnt", "bus_off_cnt", "can_core_reset_count")}} + + +def main(): + from panda import Panda + from ultimate_provisioning import UP + from openpilot.selfdrive.pandad.pandad import flash_panda + + key = inputs() + age = passed_age(key) + if age is None: + print(f"inputs {key} have no master pass in the last {PASS_TTL // 3600} h, replaying", flush=True) + elif os.environ.get("SAFETY_REPLAY_FORCE") == "1": + print(f"inputs {key} passed on master {age / 3600:.1f} h ago, replaying anyway (SAFETY_REPLAY_FORCE)", flush=True) + else: + print(f"skipped: master passed these inputs ({key}) {age / 3600:.1f} h ago, same firmware, opendbc, panda library, test and OS") + return 0 + + flash_panda() + os.environ["SAFETY_UBSAN_TRAP"] = "1" + p, up = Panda(), UP.remote() + # the UP starts with the harness lines floating + up.reset() + up.set_ignition(True) + + names = sys.argv[1:] + todo = [r for i, r in enumerate(routes) if i % NUM_JOBS == JOB_ID and (not names or r.route in names)] + failed = [] + # prepares the next route while this one replays + pool = new_pool() + loading = pool.submit(prepare, todo[0]) if todo else None + for n, route in enumerate(todo): + st = time.monotonic() + try: + prepared = loading.result() + except BrokenProcessPool as e: + # a crash in libsafety, like a UBSan trap + prepared = e + pool = new_pool() + except Exception as e: + prepared = e + if n + 1 < len(todo): + loading = pool.submit(prepare, todo[n + 1]) + if isinstance(prepared, Exception): + print(f"{route.route} FAIL, can't prepare: {type(prepared).__name__}: {prepared!r}", flush=True) + failed.append(route.route) + continue + mode, param, frames, lo, hi = prepared + got = played(p, up, mode, param, frames) + # forwarding stops at a relay malfunction, exactly when depends on the tick + if lo["relay_malfunction"] or hi["relay_malfunction"]: + for d in (lo, hi, got): + del d["fwd"] + ok = all(within(lo[k], hi[k], got[k]) for k in lo) + exp = lo if lo == hi else {"low": lo, "high": hi} + dropped = [e - g for e, g in zip(lo["rx"], got["rx"], strict=True)] + print(f"{route.route} {route.car_model} mode {mode} {'ok' if ok else 'FAIL'} {time.monotonic() - st:.0f}s expected {exp} panda {got}" + + (f" UP dropped {dropped}" if any(dropped) else ""), flush=True) + if not ok: + failed.append(route.route) + + pool.shutdown() + print(f"{len(failed)} failed: {failed}") + if failed or names: + return int(bool(failed)) + PASSED.mkdir(parents=True, exist_ok=True) + (PASSED / key).write_text(str(time.time())) # noqa: TID251 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/openpilot/selfdrive/test/diff_format.py b/openpilot/selfdrive/test/diff_format.py new file mode 100644 index 00000000000000..806867fed6cf1e --- /dev/null +++ b/openpilot/selfdrive/test/diff_format.py @@ -0,0 +1,20 @@ +# the layout of the diff report comments, light to import: CI jobs without a build use it + + +def format_report(title: str, description: str, changes: list[list[str]], n_passed: int, errors: list[str], notes: list[str] | None = None, + n_changed: int | None = None) -> str: + icon = "⚠️" if changes else "✅" + lines = [ + f"## {title}", + description, + "Please review any changes carefully to ensure they are expected.\n", + f"{icon} {len(changes) if n_changed is None else n_changed} changed, {n_passed} passed, {len(errors)} errors", + ] + lines += [f"\nERROR {e}" for e in errors] + lines += notes or [] + if changes: + lines.append("
Show changes\n\n```") + for change in changes: + lines += change + lines.append("```\n
") + return "\n".join(lines) diff --git a/openpilot/selfdrive/test/process_replay/diff_report.py b/openpilot/selfdrive/test/process_replay/diff_report.py index c6c1f01abd3c36..f91b61ea541d84 100644 --- a/openpilot/selfdrive/test/process_replay/diff_report.py +++ b/openpilot/selfdrive/test/process_replay/diff_report.py @@ -1,6 +1,7 @@ import os from collections import defaultdict from opendbc.car.tests.car_diff import format_diff, format_numeric_diffs +from openpilot.selfdrive.test.diff_format import format_report from openpilot.selfdrive.test.process_replay.compare_logs import compare_logs from openpilot.selfdrive.test.process_replay.process_replay import PROC_REPLAY_DIR @@ -66,28 +67,18 @@ def diff_report(replay_diffs, segments) -> None: else: with_diffs.append((plat, seg, proc, data)) - icon = "⚠️" if with_diffs else "✅" - lines = [ - "## Process replay diff report", - "Replays driving segments through this PR and compares the behavior to master.", - "Please review any changes carefully to ensure they are expected.\n", - f"{icon} {len(with_diffs)} changed, {n_passed} passed, {len(errors)} errors", - ] - - for plat, seg, proc, err in errors: - lines.append(f"\nERROR {plat} - {seg} [{proc}]: {err}") - - if with_diffs: - lines.append("
Show changes\n\n```") - for plat, seg, proc, (diffs, ref, new) in with_diffs: - lines.append(f"\n{plat} - {seg} [{proc}]") - by_field = defaultdict(list) - for d in diffs: - by_field[d[0]].append(d) - for field, fd in sorted(by_field.items()): - lines.append(f"\n {field} ({len(fd)} diffs)") - lines.extend(diff_format(fd, ref, new, field)) - lines.append("```\n
") + changes = [] + for plat, seg, proc, (diffs, ref, new) in with_diffs: + lines = [f"\n{plat} - {seg} [{proc}]"] + by_field = defaultdict(list) + for d in diffs: + by_field[d[0]].append(d) + for field, fd in sorted(by_field.items()): + lines.append(f"\n {field} ({len(fd)} diffs)") + lines.extend(diff_format(fd, ref, new, field)) + changes.append(lines) + report = format_report("Process replay diff report", "Replays driving segments through this PR and compares the behavior to master.", + changes, n_passed, [f"{plat} - {seg} [{proc}]: {err}" for plat, seg, proc, err in errors]) with open(os.path.join(PROC_REPLAY_DIR, "diff_report.txt"), "w") as f: - f.write("\n".join(lines)) + f.write(report) diff --git a/panda b/panda index bf14f329132082..8ae88ad1d93096 160000 --- a/panda +++ b/panda @@ -1 +1 @@ -Subproject commit bf14f329132082db7559ddca33598be7c6272528 +Subproject commit 8ae88ad1d93096c64e3e9fc143853ad7bd9ddac9