From 6c23a142300eb5d1e4038ddb9a707b11914dcdd0 Mon Sep 17 00:00:00 2001 From: elkoled Date: Thu, 5 Feb 2026 15:54:18 -0800 Subject: [PATCH 01/16] fork test --- .github/workflows/tests.yaml | 6 +++--- selfdrive/test/process_replay/test_processes.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index a0a67bcfee9036..2a63f7ede888fe 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -215,14 +215,14 @@ jobs: name: replay_diff_${{ github.event.number }} path: replay_diff.txt - name: Checkout ci-artifacts - if: github.repository == 'commaai/openpilot' && github.ref == 'refs/heads/master' + if: github.repository == 'elkoled/openpilot-stock' && github.ref == 'refs/heads/master' uses: actions/checkout@v4 with: - repository: commaai/ci-artifacts + repository: elkoled/ci-artifacts ssh-key: ${{ secrets.CI_ARTIFACTS_DEPLOY_KEY }} path: ${{ github.workspace }}/ci-artifacts - name: Push refs - if: github.repository == 'commaai/openpilot' && github.ref == 'refs/heads/master' + if: github.repository == 'elkoled/openpilot-stock' && github.ref == 'refs/heads/master' working-directory: ${{ github.workspace }}/ci-artifacts run: | git checkout --orphan process-replay diff --git a/selfdrive/test/process_replay/test_processes.py b/selfdrive/test/process_replay/test_processes.py index 9f79c8ddcb5f02..ca23cf6261a4cd 100755 --- a/selfdrive/test/process_replay/test_processes.py +++ b/selfdrive/test/process_replay/test_processes.py @@ -65,7 +65,7 @@ # dashcamOnly makes don't need to be tested until a full port is done excluded_interfaces = ["mock", "body", "psa"] -BASE_URL = "https://raw.githubusercontent.com/commaai/ci-artifacts/refs/heads/process-replay/" +BASE_URL = "https://raw.githubusercontent.com/elkoled/ci-artifacts/refs/heads/process-replay/" REF_COMMIT_FN = os.path.join(PROC_REPLAY_DIR, "ref_commit") EXCLUDED_PROCS = {"modeld", "dmonitoringmodeld"} From 7d99d7155acf45df2463d47157a6a773eb6831f2 Mon Sep 17 00:00:00 2001 From: elkoled Date: Thu, 5 Feb 2026 16:01:18 -0800 Subject: [PATCH 02/16] rename yaml --- .github/workflows/{replay_diff.yml => replay_diff.yaml} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename .github/workflows/{replay_diff.yml => replay_diff.yaml} (96%) diff --git a/.github/workflows/replay_diff.yml b/.github/workflows/replay_diff.yaml similarity index 96% rename from .github/workflows/replay_diff.yml rename to .github/workflows/replay_diff.yaml index eec1ae7f582d63..367af3a7fff0e8 100644 --- a/.github/workflows/replay_diff.yml +++ b/.github/workflows/replay_diff.yaml @@ -22,7 +22,7 @@ jobs: ref: ${{ github.event.pull_request.head.sha }} check-name: process replay repo-token: ${{ secrets.GITHUB_TOKEN }} - allowed-conclusions: success,failure + allowed-conclusions: success wait-interval: 20 - name: Download diff if: steps.wait.outcome == 'success' From a9cf3afa241eb76eab6621ad2f917b31bc48284d Mon Sep 17 00:00:00 2001 From: elkoled Date: Thu, 5 Feb 2026 17:25:40 -0800 Subject: [PATCH 03/16] integrate in test_processes --- .../{replay_diff.yaml => diff_report.yaml} | 2 +- .github/workflows/tests.yaml | 9 +- selfdrive/test/process_replay/replay_diff.py | 118 ------------------ .../test/process_replay/test_processes.py | 101 ++++++++++++++- 4 files changed, 98 insertions(+), 132 deletions(-) rename .github/workflows/{replay_diff.yaml => diff_report.yaml} (98%) delete mode 100644 selfdrive/test/process_replay/replay_diff.py diff --git a/.github/workflows/replay_diff.yaml b/.github/workflows/diff_report.yaml similarity index 98% rename from .github/workflows/replay_diff.yaml rename to .github/workflows/diff_report.yaml index 367af3a7fff0e8..d5f59ffde05727 100644 --- a/.github/workflows/replay_diff.yaml +++ b/.github/workflows/diff_report.yaml @@ -1,4 +1,4 @@ -name: replay diff +name: diff report on: pull_request_target: diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 2a63f7ede888fe..883b792eff194e 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -203,17 +203,12 @@ jobs: with: name: process_replay_diff.txt path: selfdrive/test/process_replay/diff.txt - - name: Run replay diff - if: github.event_name == 'pull_request' - timeout-minutes: 5 - run: | - ${{ env.RUN }} "python3 selfdrive/test/process_replay/replay_diff.py | tee replay_diff.txt" - uses: actions/upload-artifact@v6 if: always() && github.event_name == 'pull_request' continue-on-error: true with: - name: replay_diff_${{ github.event.number }} - path: replay_diff.txt + name: diff_report_${{ github.event.number }} + path: selfdrive/test/process_replay/diff_report.txt - name: Checkout ci-artifacts if: github.repository == 'elkoled/openpilot-stock' && github.ref == 'refs/heads/master' uses: actions/checkout@v4 diff --git a/selfdrive/test/process_replay/replay_diff.py b/selfdrive/test/process_replay/replay_diff.py deleted file mode 100644 index 1b05b74e792c77..00000000000000 --- a/selfdrive/test/process_replay/replay_diff.py +++ /dev/null @@ -1,118 +0,0 @@ -import os -import sys -from collections import defaultdict - -from opendbc.car.tests.car_diff import format_diff -from openpilot.common.git import get_commit -from openpilot.selfdrive.test.process_replay.compare_logs import compare_logs -from openpilot.selfdrive.test.process_replay.process_replay import CONFIGS, FAKEDATA, replay_process -from openpilot.selfdrive.test.process_replay.test_processes import segments, get_log_data, REF_COMMIT_FN, EXCLUDED_PROCS, BASE_URL -from openpilot.tools.lib.logreader import LogReader -from openpilot.tools.lib.url_file import URLFile - -NAN_FIELDS = {'aRel', 'yvRel'} - - -class MsgWrap: - """Adapter so to_dict() includes defaults""" - def __init__(self, msg): - self._msg = msg - def to_dict(self): - return self._msg.to_dict(verbose=True) - - -def compare_proc(cfg, ref_msgs, new_msgs): - ref, new = defaultdict(list), defaultdict(list) - for m in ref_msgs: - if m.which() in cfg.subs: - ref[m.which()].append(m) - for m in new_msgs: - if m.which() in cfg.subs: - new[m.which()].append(m) - - diffs = [] - for sub in cfg.subs: - for i, (r, n) in enumerate(zip(ref[sub], new[sub], strict=True)): - for d in compare_logs([r], [n], cfg.ignore, tolerance=cfg.tolerance): - if d[0] == "change": - path = ".".join(str(p) for p in d[1]) if isinstance(d[1], list) else d[1] - if cfg.proc_name == "card" and path.split('.')[-1] in NAN_FIELDS: - continue - diffs.append((path, i, d[2], r.logMonoTime)) - return diffs, ref, new - - -def format_proc(diffs, ref, new, field): - msg_type = field.split(".")[0] - ref = [(m.logMonoTime, MsgWrap(m)) for m in ref.get(msg_type, [])] - states = [MsgWrap(m) for m in new.get(msg_type, [])] - return format_diff(diffs, ref, states, field) - - -def main() -> int: - try: - with open(REF_COMMIT_FN) as f: - ref_commit = f.read().strip() - except FileNotFoundError: - ref_commit = URLFile(BASE_URL + "ref_commit", cache=False).read().decode().strip() - - cur_commit = get_commit() - if not cur_commit: - raise Exception("Couldn't get current commit") - - print("## Process replay diff report") - print("Replays driving segments through this PR and compares the behavior to master.") - print("Please review any changes carefully to ensure they are expected.\n") - - results = [] - for plat, seg in segments: - lr_data = None - for cfg in CONFIGS: - if cfg.proc_name in EXCLUDED_PROCS: - continue - if cfg.proc_name not in ('card', 'controlsd', 'lagd') and plat not in ('HYUNDAI', 'TOYOTA'): - continue - - cur_log_fn = os.path.join(FAKEDATA, f"{seg}_{cfg.proc_name}_{cur_commit}.zst".replace("|", "_")) - ref_fn = os.path.join(FAKEDATA, f"{seg}_{cfg.proc_name}_{ref_commit}.zst".replace("|", "_")) - ref_path = ref_fn if os.path.exists(ref_fn) else BASE_URL + os.path.basename(ref_fn) - try: - if os.path.exists(cur_log_fn): - new_msgs = list(LogReader(cur_log_fn)) - else: - if lr_data is None: - _, lr_data = get_log_data(seg) - new_msgs = replay_process(cfg, LogReader.from_bytes(lr_data), disable_progress=True) - diffs, ref, new = compare_proc(cfg, list(LogReader(ref_path)), new_msgs) - results.append((plat, seg, cfg.proc_name, (diffs, ref, new) if diffs else None, None)) - except Exception as e: - results.append((plat, seg, cfg.proc_name, None, str(e))) - - with_diffs = [(plat, seg, proc, res) for plat, seg, proc, res, err in results if res] - errors = [(plat, seg, proc, err) for plat, seg, proc, _, err in results if err] - n_passed = len(results) - len(with_diffs) - len(errors) - - icon = "⚠️" if with_diffs else "✅" - print(f"{icon} {len(with_diffs)} changed, {n_passed} passed, {len(errors)} errors") - - for plat, seg, proc, err in errors: - print(f"\nERROR {plat} - {seg} [{proc}]: {err}") - - if with_diffs: - print("
Show changes\n\n```") - for plat, seg, proc, (diffs, ref, new) in with_diffs: - print(f"\n{plat} - {seg} [{proc}]") - by_field = defaultdict(list) - for d in diffs: - by_field[d[0]].append(d) - for field, fd in sorted(by_field.items()): - print(f"\n {field} ({len(fd)} diffs)") - for line in format_proc(fd, ref, new, field): - print(line) - print("```\n
") - - return 1 if errors else 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/selfdrive/test/process_replay/test_processes.py b/selfdrive/test/process_replay/test_processes.py index ca23cf6261a4cd..279d7cfb7290f0 100755 --- a/selfdrive/test/process_replay/test_processes.py +++ b/selfdrive/test/process_replay/test_processes.py @@ -8,6 +8,7 @@ from typing import Any from opendbc.car.car_helpers import interface_names +from opendbc.car.tests.car_diff import format_diff as format_car_diff from openpilot.common.git import get_commit from openpilot.tools.lib.openpilotci import get_url from openpilot.selfdrive.test.process_replay.compare_logs import compare_logs, format_diff @@ -68,15 +69,98 @@ BASE_URL = "https://raw.githubusercontent.com/elkoled/ci-artifacts/refs/heads/process-replay/" REF_COMMIT_FN = os.path.join(PROC_REPLAY_DIR, "ref_commit") EXCLUDED_PROCS = {"modeld", "dmonitoringmodeld"} +NAN_FIELDS = {'aRel', 'yvRel'} + +# diff report +class MsgWrap: + """Adapter so to_dict() includes defaults for diff report""" + def __init__(self, msg): + self._msg = msg + def to_dict(self): + return self._msg.to_dict(verbose=True) + + +def diff_process(cfg, ref_msgs, new_msgs): + ref = defaultdict(list) + new = defaultdict(list) + for m in ref_msgs: + if m.which() in cfg.subs: + ref[m.which()].append(m) + for m in new_msgs: + if m.which() in cfg.subs: + new[m.which()].append(m) + + diffs = [] + for sub in cfg.subs: + for i, (r, n) in enumerate(zip(ref[sub], new[sub], strict=True)): + for d in compare_logs([r], [n], cfg.ignore, tolerance=cfg.tolerance): + if d[0] == "change": + path = ".".join(str(p) for p in d[1]) if isinstance(d[1], list) else d[1] + if cfg.proc_name == "card" and path.split('.')[-1] in NAN_FIELDS: + continue + diffs.append((path, i, d[2], r.logMonoTime)) + return (diffs, ref, new) if diffs else None + + +def diff_format(diffs, ref, new, field): + msg_type = field.split(".")[0] + ref_ts = [(m.logMonoTime, MsgWrap(m)) for m in ref.get(msg_type, [])] + new_wrapped = [MsgWrap(m) for m in new.get(msg_type, [])] + return format_car_diff(diffs, ref_ts, new_wrapped, field) + + +def diff_report(replay_diffs): + seg_to_plat = {seg: plat for plat, seg in segments} + + with_diffs, errors, n_passed = [], [], 0 + for seg, proc, data in replay_diffs: + plat = seg_to_plat.get(seg, "UNKNOWN") + if data is None: + n_passed += 1 + elif isinstance(data, str): + errors.append((plat, seg, proc, data)) + else: + with_diffs.append((plat, seg, proc, data)) + + icon = "⚠️" if with_diffs else "✅" + lines = [ + "## Process replay diff report", + "Replays driving segments through this PR and compares the behavior to master.", + "Please review any changes carefully to ensure they are expected.\n", + f"{icon} {len(with_diffs)} changed, {n_passed} passed, {len(errors)} errors", + ] + + for plat, seg, proc, err in errors: + lines.append(f"\nERROR {plat} - {seg} [{proc}]: {err}") + + if with_diffs: + lines.append("
Show changes\n\n```") + for plat, seg, proc, (diffs, ref, new) in with_diffs: + lines.append(f"\n{plat} - {seg} [{proc}]") + by_field = defaultdict(list) + for d in diffs: + by_field[d[0]].append(d) + for field, fd in sorted(by_field.items()): + lines.append(f"\n {field} ({len(fd)} diffs)") + lines.extend(diff_format(fd, ref, new, field)) + lines.append("```\n
") + + with open(os.path.join(PROC_REPLAY_DIR, "diff_report.txt"), "w") as f: + f.write("\n".join(lines)) def run_test_process(data): segment, cfg, args, cur_log_fn, ref_log_path, lr_dat = data + ref_log_msgs = list(LogReader(ref_log_path)) lr = LogReader.from_bytes(lr_dat) - res, log_msgs = test_process(cfg, lr, segment, ref_log_path, cur_log_fn, args.ignore_fields, args.ignore_msgs) + res, log_msgs = test_process(cfg, lr, segment, ref_log_msgs, cur_log_fn, args.ignore_fields, args.ignore_msgs) # save logs so we can update refs save_log(cur_log_fn, log_msgs) - return (segment, cfg.proc_name, res) + try: + diff_data = diff_process(cfg, ref_log_msgs, log_msgs) + except Exception: + diff_data = None + return (segment, cfg.proc_name, res, diff_data) def get_log_data(segment): @@ -85,14 +169,12 @@ def get_log_data(segment): return (segment, f.read()) -def test_process(cfg, lr, segment, ref_log_path, new_log_path, ignore_fields=None, ignore_msgs=None): +def test_process(cfg, lr, segment, ref_log_msgs, new_log_path, ignore_fields=None, ignore_msgs=None): if ignore_fields is None: ignore_fields = [] if ignore_msgs is None: ignore_msgs = [] - ref_log_msgs = list(LogReader(ref_log_path)) - try: log_msgs = replay_process(cfg, lr, disable_progress=True) except Exception as e: @@ -200,9 +282,11 @@ def test_process(cfg, lr, segment, ref_log_path, new_log_path, ignore_fields=Non log_paths[segment][cfg.proc_name]['new'] = cur_log_fn results: Any = defaultdict(dict) + diffs: list = [] p2 = pool.map(run_test_process, pool_args) - for (segment, proc, result) in tqdm(p2, desc="Running Tests", total=len(pool_args)): + for (segment, proc, result, diff_data) in tqdm(p2, desc="Running Tests", total=len(pool_args)): results[segment][proc] = result + diffs.append((segment, proc, diff_data)) diff_short, diff_long, failed = format_diff(results, log_paths, ref_commit) if not args.update_refs: @@ -210,6 +294,11 @@ def test_process(cfg, lr, segment, ref_log_path, new_log_path, ignore_fields=Non f.write(diff_long) print(diff_short) + try: + diff_report(diffs) + except Exception as e: + print(f"failed to generate diff report: {e}") + if failed: print("TEST FAILED") else: From 88b5ee198d858b814af3e1a1554e530609d1d15f Mon Sep 17 00:00:00 2001 From: elkoled Date: Thu, 5 Feb 2026 17:38:01 -0800 Subject: [PATCH 04/16] dm: suppress audible alert at standstill --- selfdrive/monitoring/helpers.py | 8 ++++++-- selfdrive/monitoring/test_monitoring.py | 12 ++++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/selfdrive/monitoring/helpers.py b/selfdrive/monitoring/helpers.py index 0b54504b647296..2243ed1b12c90c 100644 --- a/selfdrive/monitoring/helpers.py +++ b/selfdrive/monitoring/helpers.py @@ -383,8 +383,12 @@ def _update_events(self, driver_engaged, op_engaged, standstill, wrong_gear, car if awareness_prev > 0.: self.terminal_alert_cnt += 1 elif self.awareness <= self.threshold_prompt: - # prompt orange alert - alert = EventName.promptDriverDistracted if self.active_monitoring_mode else EventName.promptDriverUnresponsive + if standstill: + # green alert at standstill + alert = EventName.preDriverDistracted if self.active_monitoring_mode else EventName.preDriverUnresponsive + else: + # prompt orange alert + alert = EventName.promptDriverDistracted if self.active_monitoring_mode else EventName.promptDriverUnresponsive elif self.awareness <= self.threshold_pre and not always_on_lowspeed_exemption: # pre green alert alert = EventName.preDriverDistracted if self.active_monitoring_mode else EventName.preDriverUnresponsive diff --git a/selfdrive/monitoring/test_monitoring.py b/selfdrive/monitoring/test_monitoring.py index 6ea9b80283fc7d..3e2befe3c9a83e 100644 --- a/selfdrive/monitoring/test_monitoring.py +++ b/selfdrive/monitoring/test_monitoring.py @@ -191,6 +191,18 @@ def test_long_traffic_light_victim(self): assert events[int((_redlight_time-0.1)/DT_DMON)].names[0] == EventName.preDriverDistracted assert events[int((_redlight_time+0.5)/DT_DMON)].names[0] == EventName.promptDriverDistracted + # engaged, distracted while moving, then car stops after reaching orange + # - should downgrade to green alert when stopped, even if already in orange zone + def test_distracted_then_stops(self): + _stop_time = DISTRACTED_SECONDS_TO_ORANGE + 1 # stop 1 second after reaching orange + standstill_vector = always_false[:] + standstill_vector[int(_stop_time/DT_DMON):] = [True] * int((TEST_TIMESPAN-_stop_time)/DT_DMON) + events, _ = self._run_seq(always_distracted, always_false, always_true, standstill_vector) + # just before stopping: orange alert (car is moving, in orange zone) + assert events[int((_stop_time-0.1)/DT_DMON)].names[0] == EventName.promptDriverDistracted + # just after stopping: green alert (standstill downgrades orange to green) + assert events[int((_stop_time+0.5)/DT_DMON)].names[0] == EventName.preDriverDistracted + # engaged, model is somehow uncertain and driver is distracted # - should fall back to wheel touch after uncertain alert def test_somehow_indecisive_model(self): From 652d5b9e14341ddb1e5bf4a25f63ba9262d59c70 Mon Sep 17 00:00:00 2001 From: elkoled Date: Thu, 5 Feb 2026 18:12:11 -0800 Subject: [PATCH 05/16] fix diff report --- .github/workflows/diff_report.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/diff_report.yaml b/.github/workflows/diff_report.yaml index d5f59ffde05727..e0a42258132f7f 100644 --- a/.github/workflows/diff_report.yaml +++ b/.github/workflows/diff_report.yaml @@ -32,14 +32,14 @@ jobs: workflow: tests.yaml workflow_conclusion: '' pr: ${{ github.event.number }} - name: replay_diff_${{ github.event.number }} + name: diff_report_${{ github.event.number }} path: . allow_forks: true - name: Comment on PR if: steps.wait.outcome == 'success' uses: thollander/actions-comment-pull-request@v2 with: - filePath: replay_diff.txt + filePath: diff_report.txt comment_tag: replay_diff pr_number: ${{ github.event.number }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From c4a74704827775f2f36f9f6abeb827cd6e67ff94 Mon Sep 17 00:00:00 2001 From: elkoled Date: Thu, 5 Feb 2026 18:50:30 -0800 Subject: [PATCH 06/16] extract to module --- .github/workflows/diff_report.yaml | 2 +- selfdrive/test/process_replay/diff_report.py | 91 +++++++++++++++++++ .../test/process_replay/test_processes.py | 87 +----------------- 3 files changed, 97 insertions(+), 83 deletions(-) create mode 100644 selfdrive/test/process_replay/diff_report.py diff --git a/.github/workflows/diff_report.yaml b/.github/workflows/diff_report.yaml index e0a42258132f7f..dcd1589d416e25 100644 --- a/.github/workflows/diff_report.yaml +++ b/.github/workflows/diff_report.yaml @@ -22,7 +22,7 @@ jobs: ref: ${{ github.event.pull_request.head.sha }} check-name: process replay repo-token: ${{ secrets.GITHUB_TOKEN }} - allowed-conclusions: success + allowed-conclusions: success,failure wait-interval: 20 - name: Download diff if: steps.wait.outcome == 'success' diff --git a/selfdrive/test/process_replay/diff_report.py b/selfdrive/test/process_replay/diff_report.py new file mode 100644 index 00000000000000..76070b6be20d81 --- /dev/null +++ b/selfdrive/test/process_replay/diff_report.py @@ -0,0 +1,91 @@ +import os +from collections import defaultdict + +from opendbc.car.tests.car_diff import format_diff as format_car_diff +from openpilot.selfdrive.test.process_replay.compare_logs import compare_logs +from openpilot.selfdrive.test.process_replay.process_replay import PROC_REPLAY_DIR + +NAN_FIELDS = {'aRel', 'yvRel'} + + +class MsgWrap: + """Adapter so to_dict() includes defaults""" + def __init__(self, msg): + self._msg = msg + def to_dict(self): + return self._msg.to_dict(verbose=True) + + +def diff_process(cfg, ref_msgs, new_msgs): + ref = defaultdict(list) + new = defaultdict(list) + for m in ref_msgs: + if m.which() in cfg.subs: + ref[m.which()].append(m) + for m in new_msgs: + if m.which() in cfg.subs: + new[m.which()].append(m) + + diffs = [] + for sub in cfg.subs: + for i, (r, n) in enumerate(zip(ref[sub], new[sub], strict=True)): + for d in compare_logs([r], [n], cfg.ignore, tolerance=cfg.tolerance): + if d[0] == "change": + path = ".".join(str(p) for p in d[1]) if isinstance(d[1], list) else d[1] + if cfg.proc_name == "card" and path.split('.')[-1] in NAN_FIELDS: + continue + diffs.append((path, i, d[2], r.logMonoTime)) + elif d[0] in ("add", "remove"): + path = ".".join(str(p) for p in d[1]) if isinstance(d[1], list) else d[1] + if cfg.proc_name == "card" and path.split('.')[-1] in NAN_FIELDS: + continue + for item in d[2]: + diffs.append((f"{path}.{item[0]}", i, (d[0], item[1]), r.logMonoTime)) + return (diffs, ref, new) if diffs else None + + +def diff_format(diffs, ref, new, field): + msg_type = field.split(".")[0] + ref_ts = [(m.logMonoTime, MsgWrap(m)) for m in ref.get(msg_type, [])] + new_wrapped = [MsgWrap(m) for m in new.get(msg_type, [])] + return format_car_diff(diffs, ref_ts, new_wrapped, field) + + +def diff_report(replay_diffs, segments): + seg_to_plat = {seg: plat for plat, seg in segments} + + with_diffs, errors, n_passed = [], [], 0 + for seg, proc, data in replay_diffs: + plat = seg_to_plat.get(seg, "UNKNOWN") + if data is None: + n_passed += 1 + elif isinstance(data, str): + errors.append((plat, seg, proc, data)) + else: + with_diffs.append((plat, seg, proc, data)) + + icon = "\u26a0\ufe0f" if with_diffs else "\u2705" + lines = [ + "## Process replay diff report", + "Replays driving segments through this PR and compares the behavior to master.", + "Please review any changes carefully to ensure they are expected.\n", + f"{icon} {len(with_diffs)} changed, {n_passed} passed, {len(errors)} errors", + ] + + for plat, seg, proc, err in errors: + lines.append(f"\nERROR {plat} - {seg} [{proc}]: {err}") + + if with_diffs: + lines.append("
Show changes\n\n```") + for plat, seg, proc, (diffs, ref, new) in with_diffs: + lines.append(f"\n{plat} - {seg} [{proc}]") + by_field = defaultdict(list) + for d in diffs: + by_field[d[0]].append(d) + for field, fd in sorted(by_field.items()): + lines.append(f"\n {field} ({len(fd)} diffs)") + lines.extend(diff_format(fd, ref, new, field)) + lines.append("```\n
") + + with open(os.path.join(PROC_REPLAY_DIR, "diff_report.txt"), "w") as f: + f.write("\n".join(lines)) diff --git a/selfdrive/test/process_replay/test_processes.py b/selfdrive/test/process_replay/test_processes.py index 279d7cfb7290f0..20a227d1385004 100755 --- a/selfdrive/test/process_replay/test_processes.py +++ b/selfdrive/test/process_replay/test_processes.py @@ -3,15 +3,16 @@ import concurrent.futures import os import sys +import traceback from collections import defaultdict from tqdm import tqdm from typing import Any from opendbc.car.car_helpers import interface_names -from opendbc.car.tests.car_diff import format_diff as format_car_diff from openpilot.common.git import get_commit from openpilot.tools.lib.openpilotci import get_url from openpilot.selfdrive.test.process_replay.compare_logs import compare_logs, format_diff +from openpilot.selfdrive.test.process_replay.diff_report import diff_process, diff_report from openpilot.selfdrive.test.process_replay.process_replay import CONFIGS, PROC_REPLAY_DIR, FAKEDATA, replay_process, \ check_most_messages_valid from openpilot.tools.lib.filereader import FileReader @@ -66,87 +67,9 @@ # dashcamOnly makes don't need to be tested until a full port is done excluded_interfaces = ["mock", "body", "psa"] -BASE_URL = "https://raw.githubusercontent.com/elkoled/ci-artifacts/refs/heads/process-replay/" +BASE_URL = "https://raw.githubusercontent.com/commaai/ci-artifacts/refs/heads/process-replay/" REF_COMMIT_FN = os.path.join(PROC_REPLAY_DIR, "ref_commit") EXCLUDED_PROCS = {"modeld", "dmonitoringmodeld"} -NAN_FIELDS = {'aRel', 'yvRel'} - -# diff report -class MsgWrap: - """Adapter so to_dict() includes defaults for diff report""" - def __init__(self, msg): - self._msg = msg - def to_dict(self): - return self._msg.to_dict(verbose=True) - - -def diff_process(cfg, ref_msgs, new_msgs): - ref = defaultdict(list) - new = defaultdict(list) - for m in ref_msgs: - if m.which() in cfg.subs: - ref[m.which()].append(m) - for m in new_msgs: - if m.which() in cfg.subs: - new[m.which()].append(m) - - diffs = [] - for sub in cfg.subs: - for i, (r, n) in enumerate(zip(ref[sub], new[sub], strict=True)): - for d in compare_logs([r], [n], cfg.ignore, tolerance=cfg.tolerance): - if d[0] == "change": - path = ".".join(str(p) for p in d[1]) if isinstance(d[1], list) else d[1] - if cfg.proc_name == "card" and path.split('.')[-1] in NAN_FIELDS: - continue - diffs.append((path, i, d[2], r.logMonoTime)) - return (diffs, ref, new) if diffs else None - - -def diff_format(diffs, ref, new, field): - msg_type = field.split(".")[0] - ref_ts = [(m.logMonoTime, MsgWrap(m)) for m in ref.get(msg_type, [])] - new_wrapped = [MsgWrap(m) for m in new.get(msg_type, [])] - return format_car_diff(diffs, ref_ts, new_wrapped, field) - - -def diff_report(replay_diffs): - seg_to_plat = {seg: plat for plat, seg in segments} - - with_diffs, errors, n_passed = [], [], 0 - for seg, proc, data in replay_diffs: - plat = seg_to_plat.get(seg, "UNKNOWN") - if data is None: - n_passed += 1 - elif isinstance(data, str): - errors.append((plat, seg, proc, data)) - else: - with_diffs.append((plat, seg, proc, data)) - - icon = "⚠️" if with_diffs else "✅" - lines = [ - "## Process replay diff report", - "Replays driving segments through this PR and compares the behavior to master.", - "Please review any changes carefully to ensure they are expected.\n", - f"{icon} {len(with_diffs)} changed, {n_passed} passed, {len(errors)} errors", - ] - - for plat, seg, proc, err in errors: - lines.append(f"\nERROR {plat} - {seg} [{proc}]: {err}") - - if with_diffs: - lines.append("
Show changes\n\n```") - for plat, seg, proc, (diffs, ref, new) in with_diffs: - lines.append(f"\n{plat} - {seg} [{proc}]") - by_field = defaultdict(list) - for d in diffs: - by_field[d[0]].append(d) - for field, fd in sorted(by_field.items()): - lines.append(f"\n {field} ({len(fd)} diffs)") - lines.extend(diff_format(fd, ref, new, field)) - lines.append("```\n
") - - with open(os.path.join(PROC_REPLAY_DIR, "diff_report.txt"), "w") as f: - f.write("\n".join(lines)) def run_test_process(data): @@ -159,7 +82,7 @@ def run_test_process(data): try: diff_data = diff_process(cfg, ref_log_msgs, log_msgs) except Exception: - diff_data = None + diff_data = traceback.format_exc() return (segment, cfg.proc_name, res, diff_data) @@ -295,7 +218,7 @@ def test_process(cfg, lr, segment, ref_log_msgs, new_log_path, ignore_fields=Non print(diff_short) try: - diff_report(diffs) + diff_report(diffs, segments) except Exception as e: print(f"failed to generate diff report: {e}") From b1938b4850f8b31d774cd74b93453cce2a0e070c Mon Sep 17 00:00:00 2001 From: elkoled Date: Thu, 5 Feb 2026 18:58:42 -0800 Subject: [PATCH 07/16] elkoled --- selfdrive/test/process_replay/test_processes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/selfdrive/test/process_replay/test_processes.py b/selfdrive/test/process_replay/test_processes.py index 20a227d1385004..921ff623e2bd2d 100755 --- a/selfdrive/test/process_replay/test_processes.py +++ b/selfdrive/test/process_replay/test_processes.py @@ -67,7 +67,7 @@ # dashcamOnly makes don't need to be tested until a full port is done excluded_interfaces = ["mock", "body", "psa"] -BASE_URL = "https://raw.githubusercontent.com/commaai/ci-artifacts/refs/heads/process-replay/" +BASE_URL = "https://raw.githubusercontent.com/elkoled/ci-artifacts/refs/heads/process-replay/" REF_COMMIT_FN = os.path.join(PROC_REPLAY_DIR, "ref_commit") EXCLUDED_PROCS = {"modeld", "dmonitoringmodeld"} From d62d2df8c1a2d860367cdbdfd8d42de0b6727018 Mon Sep 17 00:00:00 2001 From: elkoled Date: Thu, 5 Feb 2026 19:07:37 -0800 Subject: [PATCH 08/16] print report --- .github/workflows/tests.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 883b792eff194e..90be560065f81f 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -197,6 +197,9 @@ jobs: id: print-diff if: always() run: cat selfdrive/test/process_replay/diff.txt + - name: Print diff report + if: always() + run: cat selfdrive/test/process_replay/diff_report.txt - uses: actions/upload-artifact@v6 if: always() continue-on-error: true From 5d0596024702dc16ae283e9610274b3d54f04382 Mon Sep 17 00:00:00 2001 From: elkoled Date: Thu, 5 Feb 2026 20:04:31 -0800 Subject: [PATCH 09/16] add msg count to diff --- selfdrive/test/process_replay/diff_report.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/selfdrive/test/process_replay/diff_report.py b/selfdrive/test/process_replay/diff_report.py index 76070b6be20d81..4611b0271a8e79 100644 --- a/selfdrive/test/process_replay/diff_report.py +++ b/selfdrive/test/process_replay/diff_report.py @@ -28,7 +28,9 @@ def diff_process(cfg, ref_msgs, new_msgs): diffs = [] for sub in cfg.subs: - for i, (r, n) in enumerate(zip(ref[sub], new[sub], strict=True)): + if len(ref[sub]) != len(new[sub]): + diffs.append((f"{sub} (message count)", 0, (len(ref[sub]), len(new[sub])), 0)) + for i, (r, n) in enumerate(zip(ref[sub], new[sub], strict=False)): for d in compare_logs([r], [n], cfg.ignore, tolerance=cfg.tolerance): if d[0] == "change": path = ".".join(str(p) for p in d[1]) if isinstance(d[1], list) else d[1] From bd7c2b4d3320008fed4949dd6b4739f549796332 Mon Sep 17 00:00:00 2001 From: elkoled Date: Thu, 5 Feb 2026 21:23:34 -0800 Subject: [PATCH 10/16] traceback --- selfdrive/test/process_replay/diff_report.py | 2 +- selfdrive/test/process_replay/test_processes.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/selfdrive/test/process_replay/diff_report.py b/selfdrive/test/process_replay/diff_report.py index 4611b0271a8e79..b4fb994e64aa06 100644 --- a/selfdrive/test/process_replay/diff_report.py +++ b/selfdrive/test/process_replay/diff_report.py @@ -66,7 +66,7 @@ def diff_report(replay_diffs, segments): else: with_diffs.append((plat, seg, proc, data)) - icon = "\u26a0\ufe0f" if with_diffs else "\u2705" + icon = "⚠️" if with_diffs else "✅" lines = [ "## Process replay diff report", "Replays driving segments through this PR and compares the behavior to master.", diff --git a/selfdrive/test/process_replay/test_processes.py b/selfdrive/test/process_replay/test_processes.py index 921ff623e2bd2d..ce9d84234a6249 100755 --- a/selfdrive/test/process_replay/test_processes.py +++ b/selfdrive/test/process_replay/test_processes.py @@ -219,8 +219,8 @@ def test_process(cfg, lr, segment, ref_log_msgs, new_log_path, ignore_fields=Non try: diff_report(diffs, segments) - except Exception as e: - print(f"failed to generate diff report: {e}") + except Exception: + print(f"failed to generate diff report:\n{traceback.format_exc()}") if failed: print("TEST FAILED") From ea188d8d2255f63ec956420f71614fedfdd37dd5 Mon Sep 17 00:00:00 2001 From: elkoled Date: Thu, 5 Feb 2026 22:34:31 -0800 Subject: [PATCH 11/16] diff format --- selfdrive/test/process_replay/diff_report.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/selfdrive/test/process_replay/diff_report.py b/selfdrive/test/process_replay/diff_report.py index b4fb994e64aa06..ce23cf7acce99a 100644 --- a/selfdrive/test/process_replay/diff_report.py +++ b/selfdrive/test/process_replay/diff_report.py @@ -1,7 +1,7 @@ import os from collections import defaultdict -from opendbc.car.tests.car_diff import format_diff as format_car_diff +from opendbc.car.tests.car_diff import format_diff, format_numeric_diffs from openpilot.selfdrive.test.process_replay.compare_logs import compare_logs from openpilot.selfdrive.test.process_replay.process_replay import PROC_REPLAY_DIR @@ -47,10 +47,12 @@ def diff_process(cfg, ref_msgs, new_msgs): def diff_format(diffs, ref, new, field): + if any(part.isdigit() for part in field.split(".")): + return format_numeric_diffs(diffs) msg_type = field.split(".")[0] ref_ts = [(m.logMonoTime, MsgWrap(m)) for m in ref.get(msg_type, [])] new_wrapped = [MsgWrap(m) for m in new.get(msg_type, [])] - return format_car_diff(diffs, ref_ts, new_wrapped, field) + return format_diff(diffs, ref_ts, new_wrapped, field) def diff_report(replay_diffs, segments): From e555c991ce9f177e00174f3f9c530472023b5809 Mon Sep 17 00:00:00 2001 From: elkoled Date: Thu, 5 Feb 2026 22:37:25 -0800 Subject: [PATCH 12/16] typing --- selfdrive/test/process_replay/diff_report.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/selfdrive/test/process_replay/diff_report.py b/selfdrive/test/process_replay/diff_report.py index ce23cf7acce99a..7b2bbab2b90a57 100644 --- a/selfdrive/test/process_replay/diff_report.py +++ b/selfdrive/test/process_replay/diff_report.py @@ -12,11 +12,11 @@ class MsgWrap: """Adapter so to_dict() includes defaults""" def __init__(self, msg): self._msg = msg - def to_dict(self): + def to_dict(self) -> dict: return self._msg.to_dict(verbose=True) -def diff_process(cfg, ref_msgs, new_msgs): +def diff_process(cfg, ref_msgs, new_msgs) -> tuple | None: ref = defaultdict(list) new = defaultdict(list) for m in ref_msgs: @@ -46,7 +46,7 @@ def diff_process(cfg, ref_msgs, new_msgs): return (diffs, ref, new) if diffs else None -def diff_format(diffs, ref, new, field): +def diff_format(diffs, ref, new, field) -> list[str]: if any(part.isdigit() for part in field.split(".")): return format_numeric_diffs(diffs) msg_type = field.split(".")[0] @@ -55,7 +55,7 @@ def diff_format(diffs, ref, new, field): return format_diff(diffs, ref_ts, new_wrapped, field) -def diff_report(replay_diffs, segments): +def diff_report(replay_diffs, segments) -> None: seg_to_plat = {seg: plat for plat, seg in segments} with_diffs, errors, n_passed = [], [], 0 From c72aa1072b108dead7cc3ca7a190a8049bd3c125 Mon Sep 17 00:00:00 2001 From: elkoled Date: Fri, 6 Feb 2026 20:16:38 -0800 Subject: [PATCH 13/16] test all processes on all segs --- selfdrive/test/process_replay/test_processes.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/selfdrive/test/process_replay/test_processes.py b/selfdrive/test/process_replay/test_processes.py index ce9d84234a6249..54af0239610000 100755 --- a/selfdrive/test/process_replay/test_processes.py +++ b/selfdrive/test/process_replay/test_processes.py @@ -188,10 +188,6 @@ def test_process(cfg, lr, segment, ref_log_msgs, new_log_path, ignore_fields=Non if cfg.proc_name not in tested_procs: continue - # to speed things up, we only test all segments on card - if cfg.proc_name not in ('card', 'controlsd', 'lagd') and car_brand not in ('HYUNDAI', 'TOYOTA'): - continue - cur_log_fn = os.path.join(FAKEDATA, f"{segment}_{cfg.proc_name}_{cur_commit}.zst".replace("|", "_")) if args.update_refs: # reference logs will not exist if routes were just regenerated ref_log_path = get_url(*segment.rsplit("--", 1,), "rlog.zst") From c4f2ab71b60b753ee648fd542cfb2f16751d19cd Mon Sep 17 00:00:00 2001 From: elkoled Date: Sun, 8 Feb 2026 22:15:56 -0800 Subject: [PATCH 14/16] bootstrap --- .github/workflows/tests.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 90be560065f81f..b6cf2480683b41 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -191,7 +191,8 @@ jobs: timeout-minutes: ${{ contains(runner.name, 'nsc') && (steps.dependency-cache.outputs.cache-hit == 'true') && ((steps.setup-step.outputs.duration < 18) && 1 || 2) || 20 }} continue-on-error: ${{ github.ref == 'refs/heads/master' }} run: | - ${{ env.RUN }} "selfdrive/test/process_replay/test_processes.py -j$(nproc) && \ + ${{ env.RUN }} "selfdrive/test/process_replay/test_processes.py -j$(nproc) \ + ${{ github.ref == 'refs/heads/master' && '--update-refs' || '' }} && \ chmod -R 777 /tmp/comma_download_cache" - name: Print diff id: print-diff @@ -213,14 +214,14 @@ jobs: name: diff_report_${{ github.event.number }} path: selfdrive/test/process_replay/diff_report.txt - name: Checkout ci-artifacts - if: github.repository == 'elkoled/openpilot-stock' && github.ref == 'refs/heads/master' + if: always() && github.repository == 'elkoled/openpilot-stock' && github.ref == 'refs/heads/master' uses: actions/checkout@v4 with: repository: elkoled/ci-artifacts ssh-key: ${{ secrets.CI_ARTIFACTS_DEPLOY_KEY }} path: ${{ github.workspace }}/ci-artifacts - name: Push refs - if: github.repository == 'elkoled/openpilot-stock' && github.ref == 'refs/heads/master' + if: always() && github.repository == 'elkoled/openpilot-stock' && github.ref == 'refs/heads/master' working-directory: ${{ github.workspace }}/ci-artifacts run: | git checkout --orphan process-replay From 7cccc4fdde0d61f76ec828d0192fc1ca2a6cd93d Mon Sep 17 00:00:00 2001 From: elkoled Date: Sun, 8 Feb 2026 22:40:36 -0800 Subject: [PATCH 15/16] Revert "bootstrap" This reverts commit c4f2ab71b60b753ee648fd542cfb2f16751d19cd. --- .github/workflows/tests.yaml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index b6cf2480683b41..90be560065f81f 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -191,8 +191,7 @@ jobs: timeout-minutes: ${{ contains(runner.name, 'nsc') && (steps.dependency-cache.outputs.cache-hit == 'true') && ((steps.setup-step.outputs.duration < 18) && 1 || 2) || 20 }} continue-on-error: ${{ github.ref == 'refs/heads/master' }} run: | - ${{ env.RUN }} "selfdrive/test/process_replay/test_processes.py -j$(nproc) \ - ${{ github.ref == 'refs/heads/master' && '--update-refs' || '' }} && \ + ${{ env.RUN }} "selfdrive/test/process_replay/test_processes.py -j$(nproc) && \ chmod -R 777 /tmp/comma_download_cache" - name: Print diff id: print-diff @@ -214,14 +213,14 @@ jobs: name: diff_report_${{ github.event.number }} path: selfdrive/test/process_replay/diff_report.txt - name: Checkout ci-artifacts - if: always() && github.repository == 'elkoled/openpilot-stock' && github.ref == 'refs/heads/master' + if: github.repository == 'elkoled/openpilot-stock' && github.ref == 'refs/heads/master' uses: actions/checkout@v4 with: repository: elkoled/ci-artifacts ssh-key: ${{ secrets.CI_ARTIFACTS_DEPLOY_KEY }} path: ${{ github.workspace }}/ci-artifacts - name: Push refs - if: always() && github.repository == 'elkoled/openpilot-stock' && github.ref == 'refs/heads/master' + if: github.repository == 'elkoled/openpilot-stock' && github.ref == 'refs/heads/master' working-directory: ${{ github.workspace }}/ci-artifacts run: | git checkout --orphan process-replay From 7315902a3b6c4837128694f848447e2c7957af2d Mon Sep 17 00:00:00 2001 From: elkoled Date: Wed, 4 Feb 2026 15:37:58 -0800 Subject: [PATCH 16/16] fork test --- .github/workflows/tests.yaml | 6 +++--- selfdrive/test/process_replay/test_processes.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 03727776bd4a9a..99aae4164eeabd 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -214,14 +214,14 @@ jobs: name: diff_report_${{ github.event.number }} path: selfdrive/test/process_replay/diff_report.txt - name: Checkout ci-artifacts - if: github.repository == 'commaai/openpilot' && github.ref == 'refs/heads/master' + if: github.repository == 'elkoled/openpilot-stock' && github.ref == 'refs/heads/master' uses: actions/checkout@v4 with: - repository: commaai/ci-artifacts + repository: elkoled/ci-artifacts ssh-key: ${{ secrets.CI_ARTIFACTS_DEPLOY_KEY }} path: ${{ github.workspace }}/ci-artifacts - name: Push refs - if: github.repository == 'commaai/openpilot' && github.ref == 'refs/heads/master' + if: github.repository == 'elkoled/openpilot-stock' && github.ref == 'refs/heads/master' working-directory: ${{ github.workspace }}/ci-artifacts run: | git checkout --orphan process-replay diff --git a/selfdrive/test/process_replay/test_processes.py b/selfdrive/test/process_replay/test_processes.py index 7c3318b6a11a44..ce9d84234a6249 100755 --- a/selfdrive/test/process_replay/test_processes.py +++ b/selfdrive/test/process_replay/test_processes.py @@ -67,7 +67,7 @@ # dashcamOnly makes don't need to be tested until a full port is done excluded_interfaces = ["mock", "body", "psa"] -BASE_URL = "https://raw.githubusercontent.com/commaai/ci-artifacts/refs/heads/process-replay/" +BASE_URL = "https://raw.githubusercontent.com/elkoled/ci-artifacts/refs/heads/process-replay/" REF_COMMIT_FN = os.path.join(PROC_REPLAY_DIR, "ref_commit") EXCLUDED_PROCS = {"modeld", "dmonitoringmodeld"}