From 6c23a142300eb5d1e4038ddb9a707b11914dcdd0 Mon Sep 17 00:00:00 2001 From: elkoled Date: Thu, 5 Feb 2026 15:54:18 -0800 Subject: [PATCH 01/16] fork test --- .github/workflows/tests.yaml | 6 +++--- selfdrive/test/process_replay/test_processes.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index a0a67bcfee9036..2a63f7ede888fe 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -215,14 +215,14 @@ jobs: name: replay_diff_${{ github.event.number }} path: replay_diff.txt - name: Checkout ci-artifacts - if: github.repository == 'commaai/openpilot' && github.ref == 'refs/heads/master' + if: github.repository == 'elkoled/openpilot-stock' && github.ref == 'refs/heads/master' uses: actions/checkout@v4 with: - repository: commaai/ci-artifacts + repository: elkoled/ci-artifacts ssh-key: ${{ secrets.CI_ARTIFACTS_DEPLOY_KEY }} path: ${{ github.workspace }}/ci-artifacts - name: Push refs - if: github.repository == 'commaai/openpilot' && github.ref == 'refs/heads/master' + if: github.repository == 'elkoled/openpilot-stock' && github.ref == 'refs/heads/master' working-directory: ${{ github.workspace }}/ci-artifacts run: | git checkout --orphan process-replay diff --git a/selfdrive/test/process_replay/test_processes.py b/selfdrive/test/process_replay/test_processes.py index 9f79c8ddcb5f02..ca23cf6261a4cd 100755 --- a/selfdrive/test/process_replay/test_processes.py +++ b/selfdrive/test/process_replay/test_processes.py @@ -65,7 +65,7 @@ # dashcamOnly makes don't need to be tested until a full port is done excluded_interfaces = ["mock", "body", "psa"] -BASE_URL = "https://raw.githubusercontent.com/commaai/ci-artifacts/refs/heads/process-replay/" +BASE_URL = "https://raw.githubusercontent.com/elkoled/ci-artifacts/refs/heads/process-replay/" REF_COMMIT_FN = os.path.join(PROC_REPLAY_DIR, "ref_commit") EXCLUDED_PROCS = {"modeld", "dmonitoringmodeld"} From 7d99d7155acf45df2463d47157a6a773eb6831f2 Mon Sep 17 00:00:00 2001 From: elkoled Date: Thu, 5 Feb 2026 16:01:18 -0800 Subject: [PATCH 02/16] rename yaml --- .github/workflows/{replay_diff.yml => replay_diff.yaml} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename .github/workflows/{replay_diff.yml => replay_diff.yaml} (96%) diff --git a/.github/workflows/replay_diff.yml b/.github/workflows/replay_diff.yaml similarity index 96% rename from .github/workflows/replay_diff.yml rename to .github/workflows/replay_diff.yaml index eec1ae7f582d63..367af3a7fff0e8 100644 --- a/.github/workflows/replay_diff.yml +++ b/.github/workflows/replay_diff.yaml @@ -22,7 +22,7 @@ jobs: ref: ${{ github.event.pull_request.head.sha }} check-name: process replay repo-token: ${{ secrets.GITHUB_TOKEN }} - allowed-conclusions: success,failure + allowed-conclusions: success wait-interval: 20 - name: Download diff if: steps.wait.outcome == 'success' From a9cf3afa241eb76eab6621ad2f917b31bc48284d Mon Sep 17 00:00:00 2001 From: elkoled Date: Thu, 5 Feb 2026 17:25:40 -0800 Subject: [PATCH 03/16] integrate in test_processes --- .../{replay_diff.yaml => diff_report.yaml} | 2 +- .github/workflows/tests.yaml | 9 +- selfdrive/test/process_replay/replay_diff.py | 118 ------------------ .../test/process_replay/test_processes.py | 101 ++++++++++++++- 4 files changed, 98 insertions(+), 132 deletions(-) rename .github/workflows/{replay_diff.yaml => diff_report.yaml} (98%) delete mode 100644 selfdrive/test/process_replay/replay_diff.py diff --git a/.github/workflows/replay_diff.yaml b/.github/workflows/diff_report.yaml similarity index 98% rename from .github/workflows/replay_diff.yaml rename to .github/workflows/diff_report.yaml index 367af3a7fff0e8..d5f59ffde05727 100644 --- a/.github/workflows/replay_diff.yaml +++ b/.github/workflows/diff_report.yaml @@ -1,4 +1,4 @@ -name: replay diff +name: diff report on: pull_request_target: diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 2a63f7ede888fe..883b792eff194e 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -203,17 +203,12 @@ jobs: with: name: process_replay_diff.txt path: selfdrive/test/process_replay/diff.txt - - name: Run replay diff - if: github.event_name == 'pull_request' - timeout-minutes: 5 - run: | - ${{ env.RUN }} "python3 selfdrive/test/process_replay/replay_diff.py | tee replay_diff.txt" - uses: actions/upload-artifact@v6 if: always() && github.event_name == 'pull_request' continue-on-error: true with: - name: replay_diff_${{ github.event.number }} - path: replay_diff.txt + name: diff_report_${{ github.event.number }} + path: selfdrive/test/process_replay/diff_report.txt - name: Checkout ci-artifacts if: github.repository == 'elkoled/openpilot-stock' && github.ref == 'refs/heads/master' uses: actions/checkout@v4 diff --git a/selfdrive/test/process_replay/replay_diff.py b/selfdrive/test/process_replay/replay_diff.py deleted file mode 100644 index 1b05b74e792c77..00000000000000 --- a/selfdrive/test/process_replay/replay_diff.py +++ /dev/null @@ -1,118 +0,0 @@ -import os -import sys -from collections import defaultdict - -from opendbc.car.tests.car_diff import format_diff -from openpilot.common.git import get_commit -from openpilot.selfdrive.test.process_replay.compare_logs import compare_logs -from openpilot.selfdrive.test.process_replay.process_replay import CONFIGS, FAKEDATA, replay_process -from openpilot.selfdrive.test.process_replay.test_processes import segments, get_log_data, REF_COMMIT_FN, EXCLUDED_PROCS, BASE_URL -from openpilot.tools.lib.logreader import LogReader -from openpilot.tools.lib.url_file import URLFile - -NAN_FIELDS = {'aRel', 'yvRel'} - - -class MsgWrap: - """Adapter so to_dict() includes defaults""" - def __init__(self, msg): - self._msg = msg - def to_dict(self): - return self._msg.to_dict(verbose=True) - - -def compare_proc(cfg, ref_msgs, new_msgs): - ref, new = defaultdict(list), defaultdict(list) - for m in ref_msgs: - if m.which() in cfg.subs: - ref[m.which()].append(m) - for m in new_msgs: - if m.which() in cfg.subs: - new[m.which()].append(m) - - diffs = [] - for sub in cfg.subs: - for i, (r, n) in enumerate(zip(ref[sub], new[sub], strict=True)): - for d in compare_logs([r], [n], cfg.ignore, tolerance=cfg.tolerance): - if d[0] == "change": - path = ".".join(str(p) for p in d[1]) if isinstance(d[1], list) else d[1] - if cfg.proc_name == "card" and path.split('.')[-1] in NAN_FIELDS: - continue - diffs.append((path, i, d[2], r.logMonoTime)) - return diffs, ref, new - - -def format_proc(diffs, ref, new, field): - msg_type = field.split(".")[0] - ref = [(m.logMonoTime, MsgWrap(m)) for m in ref.get(msg_type, [])] - states = [MsgWrap(m) for m in new.get(msg_type, [])] - return format_diff(diffs, ref, states, field) - - -def main() -> int: - try: - with open(REF_COMMIT_FN) as f: - ref_commit = f.read().strip() - except FileNotFoundError: - ref_commit = URLFile(BASE_URL + "ref_commit", cache=False).read().decode().strip() - - cur_commit = get_commit() - if not cur_commit: - raise Exception("Couldn't get current commit") - - print("## Process replay diff report") - print("Replays driving segments through this PR and compares the behavior to master.") - print("Please review any changes carefully to ensure they are expected.\n") - - results = [] - for plat, seg in segments: - lr_data = None - for cfg in CONFIGS: - if cfg.proc_name in EXCLUDED_PROCS: - continue - if cfg.proc_name not in ('card', 'controlsd', 'lagd') and plat not in ('HYUNDAI', 'TOYOTA'): - continue - - cur_log_fn = os.path.join(FAKEDATA, f"{seg}_{cfg.proc_name}_{cur_commit}.zst".replace("|", "_")) - ref_fn = os.path.join(FAKEDATA, f"{seg}_{cfg.proc_name}_{ref_commit}.zst".replace("|", "_")) - ref_path = ref_fn if os.path.exists(ref_fn) else BASE_URL + os.path.basename(ref_fn) - try: - if os.path.exists(cur_log_fn): - new_msgs = list(LogReader(cur_log_fn)) - else: - if lr_data is None: - _, lr_data = get_log_data(seg) - new_msgs = replay_process(cfg, LogReader.from_bytes(lr_data), disable_progress=True) - diffs, ref, new = compare_proc(cfg, list(LogReader(ref_path)), new_msgs) - results.append((plat, seg, cfg.proc_name, (diffs, ref, new) if diffs else None, None)) - except Exception as e: - results.append((plat, seg, cfg.proc_name, None, str(e))) - - with_diffs = [(plat, seg, proc, res) for plat, seg, proc, res, err in results if res] - errors = [(plat, seg, proc, err) for plat, seg, proc, _, err in results if err] - n_passed = len(results) - len(with_diffs) - len(errors) - - icon = "⚠️" if with_diffs else "✅" - print(f"{icon} {len(with_diffs)} changed, {n_passed} passed, {len(errors)} errors") - - for plat, seg, proc, err in errors: - print(f"\nERROR {plat} - {seg} [{proc}]: {err}") - - if with_diffs: - print("
Show changes\n\n```") - for plat, seg, proc, (diffs, ref, new) in with_diffs: - print(f"\n{plat} - {seg} [{proc}]") - by_field = defaultdict(list) - for d in diffs: - by_field[d[0]].append(d) - for field, fd in sorted(by_field.items()): - print(f"\n {field} ({len(fd)} diffs)") - for line in format_proc(fd, ref, new, field): - print(line) - print("```\n
") - - return 1 if errors else 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/selfdrive/test/process_replay/test_processes.py b/selfdrive/test/process_replay/test_processes.py index ca23cf6261a4cd..279d7cfb7290f0 100755 --- a/selfdrive/test/process_replay/test_processes.py +++ b/selfdrive/test/process_replay/test_processes.py @@ -8,6 +8,7 @@ from typing import Any from opendbc.car.car_helpers import interface_names +from opendbc.car.tests.car_diff import format_diff as format_car_diff from openpilot.common.git import get_commit from openpilot.tools.lib.openpilotci import get_url from openpilot.selfdrive.test.process_replay.compare_logs import compare_logs, format_diff @@ -68,15 +69,98 @@ BASE_URL = "https://raw.githubusercontent.com/elkoled/ci-artifacts/refs/heads/process-replay/" REF_COMMIT_FN = os.path.join(PROC_REPLAY_DIR, "ref_commit") EXCLUDED_PROCS = {"modeld", "dmonitoringmodeld"} +NAN_FIELDS = {'aRel', 'yvRel'} + +# diff report +class MsgWrap: + """Adapter so to_dict() includes defaults for diff report""" + def __init__(self, msg): + self._msg = msg + def to_dict(self): + return self._msg.to_dict(verbose=True) + + +def diff_process(cfg, ref_msgs, new_msgs): + ref = defaultdict(list) + new = defaultdict(list) + for m in ref_msgs: + if m.which() in cfg.subs: + ref[m.which()].append(m) + for m in new_msgs: + if m.which() in cfg.subs: + new[m.which()].append(m) + + diffs = [] + for sub in cfg.subs: + for i, (r, n) in enumerate(zip(ref[sub], new[sub], strict=True)): + for d in compare_logs([r], [n], cfg.ignore, tolerance=cfg.tolerance): + if d[0] == "change": + path = ".".join(str(p) for p in d[1]) if isinstance(d[1], list) else d[1] + if cfg.proc_name == "card" and path.split('.')[-1] in NAN_FIELDS: + continue + diffs.append((path, i, d[2], r.logMonoTime)) + return (diffs, ref, new) if diffs else None + + +def diff_format(diffs, ref, new, field): + msg_type = field.split(".")[0] + ref_ts = [(m.logMonoTime, MsgWrap(m)) for m in ref.get(msg_type, [])] + new_wrapped = [MsgWrap(m) for m in new.get(msg_type, [])] + return format_car_diff(diffs, ref_ts, new_wrapped, field) + + +def diff_report(replay_diffs): + seg_to_plat = {seg: plat for plat, seg in segments} + + with_diffs, errors, n_passed = [], [], 0 + for seg, proc, data in replay_diffs: + plat = seg_to_plat.get(seg, "UNKNOWN") + if data is None: + n_passed += 1 + elif isinstance(data, str): + errors.append((plat, seg, proc, data)) + else: + with_diffs.append((plat, seg, proc, data)) + + icon = "⚠️" if with_diffs else "✅" + lines = [ + "## Process replay diff report", + "Replays driving segments through this PR and compares the behavior to master.", + "Please review any changes carefully to ensure they are expected.\n", + f"{icon} {len(with_diffs)} changed, {n_passed} passed, {len(errors)} errors", + ] + + for plat, seg, proc, err in errors: + lines.append(f"\nERROR {plat} - {seg} [{proc}]: {err}") + + if with_diffs: + lines.append("
Show changes\n\n```") + for plat, seg, proc, (diffs, ref, new) in with_diffs: + lines.append(f"\n{plat} - {seg} [{proc}]") + by_field = defaultdict(list) + for d in diffs: + by_field[d[0]].append(d) + for field, fd in sorted(by_field.items()): + lines.append(f"\n {field} ({len(fd)} diffs)") + lines.extend(diff_format(fd, ref, new, field)) + lines.append("```\n
") + + with open(os.path.join(PROC_REPLAY_DIR, "diff_report.txt"), "w") as f: + f.write("\n".join(lines)) def run_test_process(data): segment, cfg, args, cur_log_fn, ref_log_path, lr_dat = data + ref_log_msgs = list(LogReader(ref_log_path)) lr = LogReader.from_bytes(lr_dat) - res, log_msgs = test_process(cfg, lr, segment, ref_log_path, cur_log_fn, args.ignore_fields, args.ignore_msgs) + res, log_msgs = test_process(cfg, lr, segment, ref_log_msgs, cur_log_fn, args.ignore_fields, args.ignore_msgs) # save logs so we can update refs save_log(cur_log_fn, log_msgs) - return (segment, cfg.proc_name, res) + try: + diff_data = diff_process(cfg, ref_log_msgs, log_msgs) + except Exception: + diff_data = None + return (segment, cfg.proc_name, res, diff_data) def get_log_data(segment): @@ -85,14 +169,12 @@ def get_log_data(segment): return (segment, f.read()) -def test_process(cfg, lr, segment, ref_log_path, new_log_path, ignore_fields=None, ignore_msgs=None): +def test_process(cfg, lr, segment, ref_log_msgs, new_log_path, ignore_fields=None, ignore_msgs=None): if ignore_fields is None: ignore_fields = [] if ignore_msgs is None: ignore_msgs = [] - ref_log_msgs = list(LogReader(ref_log_path)) - try: log_msgs = replay_process(cfg, lr, disable_progress=True) except Exception as e: @@ -200,9 +282,11 @@ def test_process(cfg, lr, segment, ref_log_path, new_log_path, ignore_fields=Non log_paths[segment][cfg.proc_name]['new'] = cur_log_fn results: Any = defaultdict(dict) + diffs: list = [] p2 = pool.map(run_test_process, pool_args) - for (segment, proc, result) in tqdm(p2, desc="Running Tests", total=len(pool_args)): + for (segment, proc, result, diff_data) in tqdm(p2, desc="Running Tests", total=len(pool_args)): results[segment][proc] = result + diffs.append((segment, proc, diff_data)) diff_short, diff_long, failed = format_diff(results, log_paths, ref_commit) if not args.update_refs: @@ -210,6 +294,11 @@ def test_process(cfg, lr, segment, ref_log_path, new_log_path, ignore_fields=Non f.write(diff_long) print(diff_short) + try: + diff_report(diffs) + except Exception as e: + print(f"failed to generate diff report: {e}") + if failed: print("TEST FAILED") else: From 063aece6b1b3e47596934278d5ad086849829139 Mon Sep 17 00:00:00 2001 From: elkoled Date: Thu, 5 Feb 2026 17:37:49 -0800 Subject: [PATCH 04/16] remove azure deps --- pyproject.toml | 2 - selfdrive/test/process_replay/regen.py | 9 +-- selfdrive/test/update_ci_routes.py | 84 ------------------------ tools/lib/azure_container.py | 73 --------------------- tools/lib/openpilotci.py | 14 +--- tools/lib/openpilotcontainers.py | 6 -- uv.lock | 88 -------------------------- 7 files changed, 5 insertions(+), 271 deletions(-) delete mode 100755 selfdrive/test/update_ci_routes.py delete mode 100644 tools/lib/azure_container.py delete mode 100755 tools/lib/openpilotcontainers.py diff --git a/pyproject.toml b/pyproject.toml index 2400364c70653b..a0d656deb8c639 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -97,8 +97,6 @@ testing = [ dev = [ "av", - "azure-identity", - "azure-storage-blob", "dictdiffer", "matplotlib", "opencv-python-headless", diff --git a/selfdrive/test/process_replay/regen.py b/selfdrive/test/process_replay/regen.py index c501a4b25083c1..6e17217c3b7a03 100755 --- a/selfdrive/test/process_replay/regen.py +++ b/selfdrive/test/process_replay/regen.py @@ -9,7 +9,6 @@ from openpilot.selfdrive.test.process_replay.process_replay import CONFIGS, FAKEDATA, ProcessConfig, replay_process, get_process_config, \ check_openpilot_enabled, check_most_messages_valid, get_custom_params_from_lr -from openpilot.selfdrive.test.update_ci_routes import upload_route from openpilot.tools.lib.framereader import FrameReader from openpilot.tools.lib.logreader import LogReader, LogIterable, save_log from openpilot.tools.lib.openpilotci import get_url @@ -52,7 +51,7 @@ def setup_data_readers( def regen_and_save( route: str, sidx: int, processes: str | Iterable[str] = "all", outdir: str = FAKEDATA, - upload: bool = False, disable_tqdm: bool = False, dummy_driver_cam: bool = False + disable_tqdm: bool = False, dummy_driver_cam: bool = False ) -> str: if not isinstance(processes, str) and not hasattr(processes, "__iter__"): raise ValueError("whitelist_proc must be a string or iterable") @@ -90,9 +89,6 @@ def regen_and_save( if not check_most_messages_valid(output_logs): raise Exception("Route has too many invalid messages") - if upload: - upload_route(rel_log_dir) - return rel_log_dir @@ -102,7 +98,6 @@ def comma_separated_list(string): all_procs = [p.proc_name for p in CONFIGS] parser = argparse.ArgumentParser(description="Generate new segments from old ones") - parser.add_argument("--upload", action="store_true", help="Upload the new segment to the CI bucket") parser.add_argument("--outdir", help="log output dir", default=FAKEDATA) parser.add_argument("--dummy-dcamera", action='store_true', help="Use dummy blank driver camera") parser.add_argument("--whitelist-procs", type=comma_separated_list, default=all_procs, @@ -115,4 +110,4 @@ def comma_separated_list(string): blacklist_set = set(args.blacklist_procs) processes = [p for p in args.whitelist_procs if p not in blacklist_set] - regen_and_save(args.route, args.seg, processes=processes, upload=args.upload, outdir=args.outdir, dummy_driver_cam=args.dummy_dcamera) + regen_and_save(args.route, args.seg, processes=processes, outdir=args.outdir, dummy_driver_cam=args.dummy_dcamera) diff --git a/selfdrive/test/update_ci_routes.py b/selfdrive/test/update_ci_routes.py deleted file mode 100755 index 54e1c88718f9f8..00000000000000 --- a/selfdrive/test/update_ci_routes.py +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/env python3 -import os -import re -import subprocess -import sys -from collections.abc import Iterable - -from tqdm import tqdm - -from opendbc.car.tests.routes import routes as test_car_models_routes -from openpilot.selfdrive.test.process_replay.test_processes import source_segments as replay_segments -from openpilot.tools.lib.azure_container import AzureContainer -from openpilot.tools.lib.openpilotcontainers import DataCIContainer, DataProdContainer, OpenpilotCIContainer - -SOURCES: list[AzureContainer] = [ - DataProdContainer, - DataCIContainer -] - -DEST = OpenpilotCIContainer - -def upload_route(path: str, exclude_patterns: Iterable[str] | None = None) -> None: - if exclude_patterns is None: - exclude_patterns = [r'dcamera\.hevc'] - - r, n = path.rsplit("--", 1) - r = '/'.join(r.split('/')[-2:]) # strip out anything extra in the path - destpath = f"{r}/{n}" - for file in os.listdir(path): - if any(re.search(pattern, file) for pattern in exclude_patterns): - continue - DEST.upload_file(os.path.join(path, file), f"{destpath}/{file}") - - -def sync_to_ci_public(route: str) -> bool: - dest_container, dest_key = DEST.get_client_and_key() - key_prefix = route.replace('|', '/') - dongle_id = key_prefix.split('/')[0] - - if next(dest_container.list_blob_names(name_starts_with=key_prefix), None) is not None: - return True - - print(f"Uploading {route}") - for source_container in SOURCES: - # assumes az login has been run - print(f"Trying {source_container.ACCOUNT}/{source_container.CONTAINER}") - _, source_key = source_container.get_client_and_key() - cmd = [ - "azcopy", - "copy", - f"{source_container.BASE_URL}{key_prefix}?{source_key}", - f"{DEST.BASE_URL}{dongle_id}?{dest_key}", - "--recursive=true", - "--overwrite=false", - "--exclude-pattern=*/dcamera.hevc", - ] - - try: - result = subprocess.call(cmd, stdout=subprocess.DEVNULL) - if result == 0: - print("Success") - return True - except subprocess.CalledProcessError: - print("Failed") - - return False - - -if __name__ == "__main__": - failed_routes = [] - - to_sync = sys.argv[1:] - - if not len(to_sync): - # sync routes from the car tests routes and process replay - to_sync.extend([rt.route for rt in test_car_models_routes]) - to_sync.extend([s[1].rsplit('--', 1)[0] for s in replay_segments]) - - for r in tqdm(to_sync): - if not sync_to_ci_public(r): - failed_routes.append(r) - - if len(failed_routes): - print("failed routes:", failed_routes) diff --git a/tools/lib/azure_container.py b/tools/lib/azure_container.py deleted file mode 100644 index 7ba2e1515fca7c..00000000000000 --- a/tools/lib/azure_container.py +++ /dev/null @@ -1,73 +0,0 @@ -import os -from datetime import datetime, timedelta, UTC -from functools import lru_cache -from pathlib import Path -from typing import IO - - -TOKEN_PATH = Path("/data/azure_token") - -@lru_cache -def get_azure_credential(): - if "AZURE_TOKEN" in os.environ: - return os.environ["AZURE_TOKEN"] - elif TOKEN_PATH.is_file(): - return TOKEN_PATH.read_text().strip() - else: - from azure.identity import AzureCliCredential - return AzureCliCredential() - -@lru_cache -def get_container_sas(account_name: str, container_name: str): - from azure.storage.blob import BlobServiceClient, ContainerSasPermissions, generate_container_sas - start_time = datetime.now(UTC).replace(tzinfo=None) - expiry_time = start_time + timedelta(hours=1) - blob_service = BlobServiceClient( - account_url=f"https://{account_name}.blob.core.windows.net", - credential=get_azure_credential(), - ) - return generate_container_sas( - account_name, - container_name, - user_delegation_key=blob_service.get_user_delegation_key(start_time, expiry_time), - permission=ContainerSasPermissions(read=True, write=True, list=True), - expiry=expiry_time, - ) - -class AzureContainer: - def __init__(self, account, container): - self.ACCOUNT = account - self.CONTAINER = container - - @property - def ACCOUNT_URL(self) -> str: - return f"https://{self.ACCOUNT}.blob.core.windows.net" - - @property - def BASE_URL(self) -> str: - return f"{self.ACCOUNT_URL}/{self.CONTAINER}/" - - def get_client_and_key(self): - from azure.storage.blob import ContainerClient - client = ContainerClient(self.ACCOUNT_URL, self.CONTAINER, credential=get_azure_credential()) - key = get_container_sas(self.ACCOUNT, self.CONTAINER) - return client, key - - def get_url(self, route_name: str, segment_num: str, filename: str) -> str: - return self.BASE_URL + f"{route_name.replace('|', '/')}/{segment_num}/{filename}" - - def upload_bytes(self, data: bytes | IO, blob_name: str, overwrite=False) -> str: - from azure.storage.blob import BlobClient - blob = BlobClient( - account_url=self.ACCOUNT_URL, - container_name=self.CONTAINER, - blob_name=blob_name, - credential=get_azure_credential(), - overwrite=overwrite, - ) - blob.upload_blob(data, overwrite=overwrite) - return self.BASE_URL + blob_name - - def upload_file(self, path: str | os.PathLike, blob_name: str, overwrite=False) -> str: - with open(path, "rb") as f: - return self.upload_bytes(f, blob_name, overwrite) diff --git a/tools/lib/openpilotci.py b/tools/lib/openpilotci.py index 1c1e1f171b7858..6b484dfdadab0a 100644 --- a/tools/lib/openpilotci.py +++ b/tools/lib/openpilotci.py @@ -1,12 +1,4 @@ -from openpilot.tools.lib.openpilotcontainers import OpenpilotCIContainer +BASE_URL = "https://commadataci.blob.core.windows.net/openpilotci/" -def get_url(*args, **kwargs): - return OpenpilotCIContainer.get_url(*args, **kwargs) - -def upload_file(*args, **kwargs): - return OpenpilotCIContainer.upload_file(*args, **kwargs) - -def upload_bytes(*args, **kwargs): - return OpenpilotCIContainer.upload_bytes(*args, **kwargs) - -BASE_URL = OpenpilotCIContainer.BASE_URL +def get_url(route_name: str, segment_num, filename: str) -> str: + return BASE_URL + f"{route_name.replace('|', '/')}/{segment_num}/{filename}" diff --git a/tools/lib/openpilotcontainers.py b/tools/lib/openpilotcontainers.py deleted file mode 100755 index 988246111189c4..00000000000000 --- a/tools/lib/openpilotcontainers.py +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env python3 -from openpilot.tools.lib.azure_container import AzureContainer - -OpenpilotCIContainer = AzureContainer("commadataci", "openpilotci") -DataCIContainer = AzureContainer("commadataci", "commadataci") -DataProdContainer = AzureContainer("commadata2", "commadata2") diff --git a/uv.lock b/uv.lock index 054a67cc9a2ccb..2cda6798946f15 100644 --- a/uv.lock +++ b/uv.lock @@ -146,50 +146,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7b/ff/48fa68888b8d5bae36d915556ff18f9e5fdc6b5ff5ae23dc4904c9713168/av-13.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:5ea0deab0e6a739cb742fba2a3983d8102f7516a3cdf3c46669f3cac0ed1f351", size = 25781343, upload-time = "2024-10-06T04:53:29.577Z" }, ] -[[package]] -name = "azure-core" -version = "1.38.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "requests" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/dc/1b/e503e08e755ea94e7d3419c9242315f888fc664211c90d032e40479022bf/azure_core-1.38.0.tar.gz", hash = "sha256:8194d2682245a3e4e3151a667c686464c3786fed7918b394d035bdcd61bb5993", size = 363033, upload-time = "2026-01-12T17:03:05.535Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fc/d8/b8fcba9464f02b121f39de2db2bf57f0b216fe11d014513d666e8634380d/azure_core-1.38.0-py3-none-any.whl", hash = "sha256:ab0c9b2cd71fecb1842d52c965c95285d3cfb38902f6766e4a471f1cd8905335", size = 217825, upload-time = "2026-01-12T17:03:07.291Z" }, -] - -[[package]] -name = "azure-identity" -version = "1.25.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "azure-core" }, - { name = "cryptography" }, - { name = "msal" }, - { name = "msal-extensions" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/06/8d/1a6c41c28a37eab26dc85ab6c86992c700cd3f4a597d9ed174b0e9c69489/azure_identity-1.25.1.tar.gz", hash = "sha256:87ca8328883de6036443e1c37b40e8dc8fb74898240f61071e09d2e369361456", size = 279826, upload-time = "2025-10-06T20:30:02.194Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/83/7b/5652771e24fff12da9dde4c20ecf4682e606b104f26419d139758cc935a6/azure_identity-1.25.1-py3-none-any.whl", hash = "sha256:e9edd720af03dff020223cd269fa3a61e8f345ea75443858273bcb44844ab651", size = 191317, upload-time = "2025-10-06T20:30:04.251Z" }, -] - -[[package]] -name = "azure-storage-blob" -version = "12.28.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "azure-core" }, - { name = "cryptography" }, - { name = "isodate" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/71/24/072ba8e27b0e2d8fec401e9969b429d4f5fc4c8d4f0f05f4661e11f7234a/azure_storage_blob-12.28.0.tar.gz", hash = "sha256:e7d98ea108258d29aa0efbfd591b2e2075fa1722a2fae8699f0b3c9de11eff41", size = 604225, upload-time = "2026-01-06T23:48:57.282Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d8/3a/6ef2047a072e54e1142718d433d50e9514c999a58f51abfff7902f3a72f8/azure_storage_blob-12.28.0-py3-none-any.whl", hash = "sha256:00fb1db28bf6a7b7ecaa48e3b1d5c83bfadacc5a678b77826081304bd87d6461", size = 431499, upload-time = "2026-01-06T23:48:58.995Z" }, -] - [[package]] name = "casadi" version = "3.7.2" @@ -729,15 +685,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d0/94/040a0d9c81f018c39bd887b7b825013b024deb0a6c795f9524797e2cd41b/inputs-0.5-py2.py3-none-any.whl", hash = "sha256:13f894564e52134cf1e3862b1811da034875eb1f2b62e6021e3776e9669a96ec", size = 33630, upload-time = "2018-10-05T22:38:28.28Z" }, ] -[[package]] -name = "isodate" -version = "0.7.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/54/4d/e940025e2ce31a8ce1202635910747e5a87cc3a6a6bb2d00973375014749/isodate-0.7.2.tar.gz", hash = "sha256:4cd1aa0f43ca76f4a6c6c0292a85f40b35ec2e43e315b59f06e6d32171a953e6", size = 29705, upload-time = "2024-10-08T23:04:11.5Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/15/aa/0aca39a37d3c7eb941ba736ede56d689e7be91cab5d9ca846bde3999eba6/isodate-0.7.2-py3-none-any.whl", hash = "sha256:28009937d8031054830160fce6d409ed342816b543597cece116d966c6d99e15", size = 22320, upload-time = "2024-10-08T23:04:09.501Z" }, -] - [[package]] name = "jeepney" version = "0.9.0" @@ -1112,32 +1059,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/43/e3/7d92a15f894aa0c9c4b49b8ee9ac9850d6e63b03c9c32c0367a13ae62209/mpmath-1.3.0-py3-none-any.whl", hash = "sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c", size = 536198, upload-time = "2023-03-07T16:47:09.197Z" }, ] -[[package]] -name = "msal" -version = "1.34.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cryptography" }, - { name = "pyjwt", extra = ["crypto"] }, - { name = "requests" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/cf/0e/c857c46d653e104019a84f22d4494f2119b4fe9f896c92b4b864b3b045cc/msal-1.34.0.tar.gz", hash = "sha256:76ba83b716ea5a6d75b0279c0ac353a0e05b820ca1f6682c0eb7f45190c43c2f", size = 153961, upload-time = "2025-09-22T23:05:48.989Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c2/dc/18d48843499e278538890dc709e9ee3dea8375f8be8e82682851df1b48b5/msal-1.34.0-py3-none-any.whl", hash = "sha256:f669b1644e4950115da7a176441b0e13ec2975c29528d8b9e81316023676d6e1", size = 116987, upload-time = "2025-09-22T23:05:47.294Z" }, -] - -[[package]] -name = "msal-extensions" -version = "1.3.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "msal" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/01/99/5d239b6156eddf761a636bded1118414d161bd6b7b37a9335549ed159396/msal_extensions-1.3.1.tar.gz", hash = "sha256:c5b0fd10f65ef62b5f1d62f4251d51cbcaf003fcedae8c91b040a488614be1a4", size = 23315, upload-time = "2025-03-14T23:51:03.902Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5e/75/bd9b7bb966668920f06b200e84454c8f3566b102183bc55c5473d96cb2b9/msal_extensions-1.3.1-py3-none-any.whl", hash = "sha256:96d3de4d034504e969ac5e85bae8106c8373b5c6568e4c8fa7af2eca9dbe6bca", size = 20583, upload-time = "2025-03-14T23:51:03.016Z" }, -] - [[package]] name = "multidict" version = "6.7.1" @@ -1309,8 +1230,6 @@ dependencies = [ [package.optional-dependencies] dev = [ { name = "av" }, - { name = "azure-identity" }, - { name = "azure-storage-blob" }, { name = "dictdiffer" }, { name = "matplotlib" }, { name = "opencv-python-headless" }, @@ -1349,8 +1268,6 @@ requires-dist = [ { name = "aiohttp" }, { name = "aiortc" }, { name = "av", marker = "extra == 'dev'" }, - { name = "azure-identity", marker = "extra == 'dev'" }, - { name = "azure-storage-blob", marker = "extra == 'dev'" }, { name = "casadi", specifier = ">=3.6.6" }, { name = "cffi" }, { name = "codespell", marker = "extra == 'testing'" }, @@ -1753,11 +1670,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/61/ad/689f02752eeec26aed679477e80e632ef1b682313be70793d798c1d5fc8f/PyJWT-2.10.1-py3-none-any.whl", hash = "sha256:dcdd193e30abefd5debf142f9adfcdd2b58004e644f25406ffaebd50bd98dacb", size = 22997, upload-time = "2024-11-28T03:43:27.893Z" }, ] -[package.optional-dependencies] -crypto = [ - { name = "cryptography" }, -] - [[package]] name = "pylibsrtp" version = "1.0.0" From 652d5b9e14341ddb1e5bf4a25f63ba9262d59c70 Mon Sep 17 00:00:00 2001 From: elkoled Date: Thu, 5 Feb 2026 18:12:11 -0800 Subject: [PATCH 05/16] fix diff report --- .github/workflows/diff_report.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/diff_report.yaml b/.github/workflows/diff_report.yaml index d5f59ffde05727..e0a42258132f7f 100644 --- a/.github/workflows/diff_report.yaml +++ b/.github/workflows/diff_report.yaml @@ -32,14 +32,14 @@ jobs: workflow: tests.yaml workflow_conclusion: '' pr: ${{ github.event.number }} - name: replay_diff_${{ github.event.number }} + name: diff_report_${{ github.event.number }} path: . allow_forks: true - name: Comment on PR if: steps.wait.outcome == 'success' uses: thollander/actions-comment-pull-request@v2 with: - filePath: replay_diff.txt + filePath: diff_report.txt comment_tag: replay_diff pr_number: ${{ github.event.number }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From c4a74704827775f2f36f9f6abeb827cd6e67ff94 Mon Sep 17 00:00:00 2001 From: elkoled Date: Thu, 5 Feb 2026 18:50:30 -0800 Subject: [PATCH 06/16] extract to module --- .github/workflows/diff_report.yaml | 2 +- selfdrive/test/process_replay/diff_report.py | 91 +++++++++++++++++++ .../test/process_replay/test_processes.py | 87 +----------------- 3 files changed, 97 insertions(+), 83 deletions(-) create mode 100644 selfdrive/test/process_replay/diff_report.py diff --git a/.github/workflows/diff_report.yaml b/.github/workflows/diff_report.yaml index e0a42258132f7f..dcd1589d416e25 100644 --- a/.github/workflows/diff_report.yaml +++ b/.github/workflows/diff_report.yaml @@ -22,7 +22,7 @@ jobs: ref: ${{ github.event.pull_request.head.sha }} check-name: process replay repo-token: ${{ secrets.GITHUB_TOKEN }} - allowed-conclusions: success + allowed-conclusions: success,failure wait-interval: 20 - name: Download diff if: steps.wait.outcome == 'success' diff --git a/selfdrive/test/process_replay/diff_report.py b/selfdrive/test/process_replay/diff_report.py new file mode 100644 index 00000000000000..76070b6be20d81 --- /dev/null +++ b/selfdrive/test/process_replay/diff_report.py @@ -0,0 +1,91 @@ +import os +from collections import defaultdict + +from opendbc.car.tests.car_diff import format_diff as format_car_diff +from openpilot.selfdrive.test.process_replay.compare_logs import compare_logs +from openpilot.selfdrive.test.process_replay.process_replay import PROC_REPLAY_DIR + +NAN_FIELDS = {'aRel', 'yvRel'} + + +class MsgWrap: + """Adapter so to_dict() includes defaults""" + def __init__(self, msg): + self._msg = msg + def to_dict(self): + return self._msg.to_dict(verbose=True) + + +def diff_process(cfg, ref_msgs, new_msgs): + ref = defaultdict(list) + new = defaultdict(list) + for m in ref_msgs: + if m.which() in cfg.subs: + ref[m.which()].append(m) + for m in new_msgs: + if m.which() in cfg.subs: + new[m.which()].append(m) + + diffs = [] + for sub in cfg.subs: + for i, (r, n) in enumerate(zip(ref[sub], new[sub], strict=True)): + for d in compare_logs([r], [n], cfg.ignore, tolerance=cfg.tolerance): + if d[0] == "change": + path = ".".join(str(p) for p in d[1]) if isinstance(d[1], list) else d[1] + if cfg.proc_name == "card" and path.split('.')[-1] in NAN_FIELDS: + continue + diffs.append((path, i, d[2], r.logMonoTime)) + elif d[0] in ("add", "remove"): + path = ".".join(str(p) for p in d[1]) if isinstance(d[1], list) else d[1] + if cfg.proc_name == "card" and path.split('.')[-1] in NAN_FIELDS: + continue + for item in d[2]: + diffs.append((f"{path}.{item[0]}", i, (d[0], item[1]), r.logMonoTime)) + return (diffs, ref, new) if diffs else None + + +def diff_format(diffs, ref, new, field): + msg_type = field.split(".")[0] + ref_ts = [(m.logMonoTime, MsgWrap(m)) for m in ref.get(msg_type, [])] + new_wrapped = [MsgWrap(m) for m in new.get(msg_type, [])] + return format_car_diff(diffs, ref_ts, new_wrapped, field) + + +def diff_report(replay_diffs, segments): + seg_to_plat = {seg: plat for plat, seg in segments} + + with_diffs, errors, n_passed = [], [], 0 + for seg, proc, data in replay_diffs: + plat = seg_to_plat.get(seg, "UNKNOWN") + if data is None: + n_passed += 1 + elif isinstance(data, str): + errors.append((plat, seg, proc, data)) + else: + with_diffs.append((plat, seg, proc, data)) + + icon = "\u26a0\ufe0f" if with_diffs else "\u2705" + lines = [ + "## Process replay diff report", + "Replays driving segments through this PR and compares the behavior to master.", + "Please review any changes carefully to ensure they are expected.\n", + f"{icon} {len(with_diffs)} changed, {n_passed} passed, {len(errors)} errors", + ] + + for plat, seg, proc, err in errors: + lines.append(f"\nERROR {plat} - {seg} [{proc}]: {err}") + + if with_diffs: + lines.append("
Show changes\n\n```") + for plat, seg, proc, (diffs, ref, new) in with_diffs: + lines.append(f"\n{plat} - {seg} [{proc}]") + by_field = defaultdict(list) + for d in diffs: + by_field[d[0]].append(d) + for field, fd in sorted(by_field.items()): + lines.append(f"\n {field} ({len(fd)} diffs)") + lines.extend(diff_format(fd, ref, new, field)) + lines.append("```\n
") + + with open(os.path.join(PROC_REPLAY_DIR, "diff_report.txt"), "w") as f: + f.write("\n".join(lines)) diff --git a/selfdrive/test/process_replay/test_processes.py b/selfdrive/test/process_replay/test_processes.py index 279d7cfb7290f0..20a227d1385004 100755 --- a/selfdrive/test/process_replay/test_processes.py +++ b/selfdrive/test/process_replay/test_processes.py @@ -3,15 +3,16 @@ import concurrent.futures import os import sys +import traceback from collections import defaultdict from tqdm import tqdm from typing import Any from opendbc.car.car_helpers import interface_names -from opendbc.car.tests.car_diff import format_diff as format_car_diff from openpilot.common.git import get_commit from openpilot.tools.lib.openpilotci import get_url from openpilot.selfdrive.test.process_replay.compare_logs import compare_logs, format_diff +from openpilot.selfdrive.test.process_replay.diff_report import diff_process, diff_report from openpilot.selfdrive.test.process_replay.process_replay import CONFIGS, PROC_REPLAY_DIR, FAKEDATA, replay_process, \ check_most_messages_valid from openpilot.tools.lib.filereader import FileReader @@ -66,87 +67,9 @@ # dashcamOnly makes don't need to be tested until a full port is done excluded_interfaces = ["mock", "body", "psa"] -BASE_URL = "https://raw.githubusercontent.com/elkoled/ci-artifacts/refs/heads/process-replay/" +BASE_URL = "https://raw.githubusercontent.com/commaai/ci-artifacts/refs/heads/process-replay/" REF_COMMIT_FN = os.path.join(PROC_REPLAY_DIR, "ref_commit") EXCLUDED_PROCS = {"modeld", "dmonitoringmodeld"} -NAN_FIELDS = {'aRel', 'yvRel'} - -# diff report -class MsgWrap: - """Adapter so to_dict() includes defaults for diff report""" - def __init__(self, msg): - self._msg = msg - def to_dict(self): - return self._msg.to_dict(verbose=True) - - -def diff_process(cfg, ref_msgs, new_msgs): - ref = defaultdict(list) - new = defaultdict(list) - for m in ref_msgs: - if m.which() in cfg.subs: - ref[m.which()].append(m) - for m in new_msgs: - if m.which() in cfg.subs: - new[m.which()].append(m) - - diffs = [] - for sub in cfg.subs: - for i, (r, n) in enumerate(zip(ref[sub], new[sub], strict=True)): - for d in compare_logs([r], [n], cfg.ignore, tolerance=cfg.tolerance): - if d[0] == "change": - path = ".".join(str(p) for p in d[1]) if isinstance(d[1], list) else d[1] - if cfg.proc_name == "card" and path.split('.')[-1] in NAN_FIELDS: - continue - diffs.append((path, i, d[2], r.logMonoTime)) - return (diffs, ref, new) if diffs else None - - -def diff_format(diffs, ref, new, field): - msg_type = field.split(".")[0] - ref_ts = [(m.logMonoTime, MsgWrap(m)) for m in ref.get(msg_type, [])] - new_wrapped = [MsgWrap(m) for m in new.get(msg_type, [])] - return format_car_diff(diffs, ref_ts, new_wrapped, field) - - -def diff_report(replay_diffs): - seg_to_plat = {seg: plat for plat, seg in segments} - - with_diffs, errors, n_passed = [], [], 0 - for seg, proc, data in replay_diffs: - plat = seg_to_plat.get(seg, "UNKNOWN") - if data is None: - n_passed += 1 - elif isinstance(data, str): - errors.append((plat, seg, proc, data)) - else: - with_diffs.append((plat, seg, proc, data)) - - icon = "⚠️" if with_diffs else "✅" - lines = [ - "## Process replay diff report", - "Replays driving segments through this PR and compares the behavior to master.", - "Please review any changes carefully to ensure they are expected.\n", - f"{icon} {len(with_diffs)} changed, {n_passed} passed, {len(errors)} errors", - ] - - for plat, seg, proc, err in errors: - lines.append(f"\nERROR {plat} - {seg} [{proc}]: {err}") - - if with_diffs: - lines.append("
Show changes\n\n```") - for plat, seg, proc, (diffs, ref, new) in with_diffs: - lines.append(f"\n{plat} - {seg} [{proc}]") - by_field = defaultdict(list) - for d in diffs: - by_field[d[0]].append(d) - for field, fd in sorted(by_field.items()): - lines.append(f"\n {field} ({len(fd)} diffs)") - lines.extend(diff_format(fd, ref, new, field)) - lines.append("```\n
") - - with open(os.path.join(PROC_REPLAY_DIR, "diff_report.txt"), "w") as f: - f.write("\n".join(lines)) def run_test_process(data): @@ -159,7 +82,7 @@ def run_test_process(data): try: diff_data = diff_process(cfg, ref_log_msgs, log_msgs) except Exception: - diff_data = None + diff_data = traceback.format_exc() return (segment, cfg.proc_name, res, diff_data) @@ -295,7 +218,7 @@ def test_process(cfg, lr, segment, ref_log_msgs, new_log_path, ignore_fields=Non print(diff_short) try: - diff_report(diffs) + diff_report(diffs, segments) except Exception as e: print(f"failed to generate diff report: {e}") From b1938b4850f8b31d774cd74b93453cce2a0e070c Mon Sep 17 00:00:00 2001 From: elkoled Date: Thu, 5 Feb 2026 18:58:42 -0800 Subject: [PATCH 07/16] elkoled --- selfdrive/test/process_replay/test_processes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/selfdrive/test/process_replay/test_processes.py b/selfdrive/test/process_replay/test_processes.py index 20a227d1385004..921ff623e2bd2d 100755 --- a/selfdrive/test/process_replay/test_processes.py +++ b/selfdrive/test/process_replay/test_processes.py @@ -67,7 +67,7 @@ # dashcamOnly makes don't need to be tested until a full port is done excluded_interfaces = ["mock", "body", "psa"] -BASE_URL = "https://raw.githubusercontent.com/commaai/ci-artifacts/refs/heads/process-replay/" +BASE_URL = "https://raw.githubusercontent.com/elkoled/ci-artifacts/refs/heads/process-replay/" REF_COMMIT_FN = os.path.join(PROC_REPLAY_DIR, "ref_commit") EXCLUDED_PROCS = {"modeld", "dmonitoringmodeld"} From d62d2df8c1a2d860367cdbdfd8d42de0b6727018 Mon Sep 17 00:00:00 2001 From: elkoled Date: Thu, 5 Feb 2026 19:07:37 -0800 Subject: [PATCH 08/16] print report --- .github/workflows/tests.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 883b792eff194e..90be560065f81f 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -197,6 +197,9 @@ jobs: id: print-diff if: always() run: cat selfdrive/test/process_replay/diff.txt + - name: Print diff report + if: always() + run: cat selfdrive/test/process_replay/diff_report.txt - uses: actions/upload-artifact@v6 if: always() continue-on-error: true From 5d0596024702dc16ae283e9610274b3d54f04382 Mon Sep 17 00:00:00 2001 From: elkoled Date: Thu, 5 Feb 2026 20:04:31 -0800 Subject: [PATCH 09/16] add msg count to diff --- selfdrive/test/process_replay/diff_report.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/selfdrive/test/process_replay/diff_report.py b/selfdrive/test/process_replay/diff_report.py index 76070b6be20d81..4611b0271a8e79 100644 --- a/selfdrive/test/process_replay/diff_report.py +++ b/selfdrive/test/process_replay/diff_report.py @@ -28,7 +28,9 @@ def diff_process(cfg, ref_msgs, new_msgs): diffs = [] for sub in cfg.subs: - for i, (r, n) in enumerate(zip(ref[sub], new[sub], strict=True)): + if len(ref[sub]) != len(new[sub]): + diffs.append((f"{sub} (message count)", 0, (len(ref[sub]), len(new[sub])), 0)) + for i, (r, n) in enumerate(zip(ref[sub], new[sub], strict=False)): for d in compare_logs([r], [n], cfg.ignore, tolerance=cfg.tolerance): if d[0] == "change": path = ".".join(str(p) for p in d[1]) if isinstance(d[1], list) else d[1] From bd7c2b4d3320008fed4949dd6b4739f549796332 Mon Sep 17 00:00:00 2001 From: elkoled Date: Thu, 5 Feb 2026 21:23:34 -0800 Subject: [PATCH 10/16] traceback --- selfdrive/test/process_replay/diff_report.py | 2 +- selfdrive/test/process_replay/test_processes.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/selfdrive/test/process_replay/diff_report.py b/selfdrive/test/process_replay/diff_report.py index 4611b0271a8e79..b4fb994e64aa06 100644 --- a/selfdrive/test/process_replay/diff_report.py +++ b/selfdrive/test/process_replay/diff_report.py @@ -66,7 +66,7 @@ def diff_report(replay_diffs, segments): else: with_diffs.append((plat, seg, proc, data)) - icon = "\u26a0\ufe0f" if with_diffs else "\u2705" + icon = "⚠️" if with_diffs else "✅" lines = [ "## Process replay diff report", "Replays driving segments through this PR and compares the behavior to master.", diff --git a/selfdrive/test/process_replay/test_processes.py b/selfdrive/test/process_replay/test_processes.py index 921ff623e2bd2d..ce9d84234a6249 100755 --- a/selfdrive/test/process_replay/test_processes.py +++ b/selfdrive/test/process_replay/test_processes.py @@ -219,8 +219,8 @@ def test_process(cfg, lr, segment, ref_log_msgs, new_log_path, ignore_fields=Non try: diff_report(diffs, segments) - except Exception as e: - print(f"failed to generate diff report: {e}") + except Exception: + print(f"failed to generate diff report:\n{traceback.format_exc()}") if failed: print("TEST FAILED") From ea188d8d2255f63ec956420f71614fedfdd37dd5 Mon Sep 17 00:00:00 2001 From: elkoled Date: Thu, 5 Feb 2026 22:34:31 -0800 Subject: [PATCH 11/16] diff format --- selfdrive/test/process_replay/diff_report.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/selfdrive/test/process_replay/diff_report.py b/selfdrive/test/process_replay/diff_report.py index b4fb994e64aa06..ce23cf7acce99a 100644 --- a/selfdrive/test/process_replay/diff_report.py +++ b/selfdrive/test/process_replay/diff_report.py @@ -1,7 +1,7 @@ import os from collections import defaultdict -from opendbc.car.tests.car_diff import format_diff as format_car_diff +from opendbc.car.tests.car_diff import format_diff, format_numeric_diffs from openpilot.selfdrive.test.process_replay.compare_logs import compare_logs from openpilot.selfdrive.test.process_replay.process_replay import PROC_REPLAY_DIR @@ -47,10 +47,12 @@ def diff_process(cfg, ref_msgs, new_msgs): def diff_format(diffs, ref, new, field): + if any(part.isdigit() for part in field.split(".")): + return format_numeric_diffs(diffs) msg_type = field.split(".")[0] ref_ts = [(m.logMonoTime, MsgWrap(m)) for m in ref.get(msg_type, [])] new_wrapped = [MsgWrap(m) for m in new.get(msg_type, [])] - return format_car_diff(diffs, ref_ts, new_wrapped, field) + return format_diff(diffs, ref_ts, new_wrapped, field) def diff_report(replay_diffs, segments): From e555c991ce9f177e00174f3f9c530472023b5809 Mon Sep 17 00:00:00 2001 From: elkoled Date: Thu, 5 Feb 2026 22:37:25 -0800 Subject: [PATCH 12/16] typing --- selfdrive/test/process_replay/diff_report.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/selfdrive/test/process_replay/diff_report.py b/selfdrive/test/process_replay/diff_report.py index ce23cf7acce99a..7b2bbab2b90a57 100644 --- a/selfdrive/test/process_replay/diff_report.py +++ b/selfdrive/test/process_replay/diff_report.py @@ -12,11 +12,11 @@ class MsgWrap: """Adapter so to_dict() includes defaults""" def __init__(self, msg): self._msg = msg - def to_dict(self): + def to_dict(self) -> dict: return self._msg.to_dict(verbose=True) -def diff_process(cfg, ref_msgs, new_msgs): +def diff_process(cfg, ref_msgs, new_msgs) -> tuple | None: ref = defaultdict(list) new = defaultdict(list) for m in ref_msgs: @@ -46,7 +46,7 @@ def diff_process(cfg, ref_msgs, new_msgs): return (diffs, ref, new) if diffs else None -def diff_format(diffs, ref, new, field): +def diff_format(diffs, ref, new, field) -> list[str]: if any(part.isdigit() for part in field.split(".")): return format_numeric_diffs(diffs) msg_type = field.split(".")[0] @@ -55,7 +55,7 @@ def diff_format(diffs, ref, new, field): return format_diff(diffs, ref_ts, new_wrapped, field) -def diff_report(replay_diffs, segments): +def diff_report(replay_diffs, segments) -> None: seg_to_plat = {seg: plat for plat, seg in segments} with_diffs, errors, n_passed = [], [], 0 From c72aa1072b108dead7cc3ca7a190a8049bd3c125 Mon Sep 17 00:00:00 2001 From: elkoled Date: Fri, 6 Feb 2026 20:16:38 -0800 Subject: [PATCH 13/16] test all processes on all segs --- selfdrive/test/process_replay/test_processes.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/selfdrive/test/process_replay/test_processes.py b/selfdrive/test/process_replay/test_processes.py index ce9d84234a6249..54af0239610000 100755 --- a/selfdrive/test/process_replay/test_processes.py +++ b/selfdrive/test/process_replay/test_processes.py @@ -188,10 +188,6 @@ def test_process(cfg, lr, segment, ref_log_msgs, new_log_path, ignore_fields=Non if cfg.proc_name not in tested_procs: continue - # to speed things up, we only test all segments on card - if cfg.proc_name not in ('card', 'controlsd', 'lagd') and car_brand not in ('HYUNDAI', 'TOYOTA'): - continue - cur_log_fn = os.path.join(FAKEDATA, f"{segment}_{cfg.proc_name}_{cur_commit}.zst".replace("|", "_")) if args.update_refs: # reference logs will not exist if routes were just regenerated ref_log_path = get_url(*segment.rsplit("--", 1,), "rlog.zst") From c4f2ab71b60b753ee648fd542cfb2f16751d19cd Mon Sep 17 00:00:00 2001 From: elkoled Date: Sun, 8 Feb 2026 22:15:56 -0800 Subject: [PATCH 14/16] bootstrap --- .github/workflows/tests.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 90be560065f81f..b6cf2480683b41 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -191,7 +191,8 @@ jobs: timeout-minutes: ${{ contains(runner.name, 'nsc') && (steps.dependency-cache.outputs.cache-hit == 'true') && ((steps.setup-step.outputs.duration < 18) && 1 || 2) || 20 }} continue-on-error: ${{ github.ref == 'refs/heads/master' }} run: | - ${{ env.RUN }} "selfdrive/test/process_replay/test_processes.py -j$(nproc) && \ + ${{ env.RUN }} "selfdrive/test/process_replay/test_processes.py -j$(nproc) \ + ${{ github.ref == 'refs/heads/master' && '--update-refs' || '' }} && \ chmod -R 777 /tmp/comma_download_cache" - name: Print diff id: print-diff @@ -213,14 +214,14 @@ jobs: name: diff_report_${{ github.event.number }} path: selfdrive/test/process_replay/diff_report.txt - name: Checkout ci-artifacts - if: github.repository == 'elkoled/openpilot-stock' && github.ref == 'refs/heads/master' + if: always() && github.repository == 'elkoled/openpilot-stock' && github.ref == 'refs/heads/master' uses: actions/checkout@v4 with: repository: elkoled/ci-artifacts ssh-key: ${{ secrets.CI_ARTIFACTS_DEPLOY_KEY }} path: ${{ github.workspace }}/ci-artifacts - name: Push refs - if: github.repository == 'elkoled/openpilot-stock' && github.ref == 'refs/heads/master' + if: always() && github.repository == 'elkoled/openpilot-stock' && github.ref == 'refs/heads/master' working-directory: ${{ github.workspace }}/ci-artifacts run: | git checkout --orphan process-replay From 7cccc4fdde0d61f76ec828d0192fc1ca2a6cd93d Mon Sep 17 00:00:00 2001 From: elkoled Date: Sun, 8 Feb 2026 22:40:36 -0800 Subject: [PATCH 15/16] Revert "bootstrap" This reverts commit c4f2ab71b60b753ee648fd542cfb2f16751d19cd. --- .github/workflows/tests.yaml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index b6cf2480683b41..90be560065f81f 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -191,8 +191,7 @@ jobs: timeout-minutes: ${{ contains(runner.name, 'nsc') && (steps.dependency-cache.outputs.cache-hit == 'true') && ((steps.setup-step.outputs.duration < 18) && 1 || 2) || 20 }} continue-on-error: ${{ github.ref == 'refs/heads/master' }} run: | - ${{ env.RUN }} "selfdrive/test/process_replay/test_processes.py -j$(nproc) \ - ${{ github.ref == 'refs/heads/master' && '--update-refs' || '' }} && \ + ${{ env.RUN }} "selfdrive/test/process_replay/test_processes.py -j$(nproc) && \ chmod -R 777 /tmp/comma_download_cache" - name: Print diff id: print-diff @@ -214,14 +213,14 @@ jobs: name: diff_report_${{ github.event.number }} path: selfdrive/test/process_replay/diff_report.txt - name: Checkout ci-artifacts - if: always() && github.repository == 'elkoled/openpilot-stock' && github.ref == 'refs/heads/master' + if: github.repository == 'elkoled/openpilot-stock' && github.ref == 'refs/heads/master' uses: actions/checkout@v4 with: repository: elkoled/ci-artifacts ssh-key: ${{ secrets.CI_ARTIFACTS_DEPLOY_KEY }} path: ${{ github.workspace }}/ci-artifacts - name: Push refs - if: always() && github.repository == 'elkoled/openpilot-stock' && github.ref == 'refs/heads/master' + if: github.repository == 'elkoled/openpilot-stock' && github.ref == 'refs/heads/master' working-directory: ${{ github.workspace }}/ci-artifacts run: | git checkout --orphan process-replay From 7315902a3b6c4837128694f848447e2c7957af2d Mon Sep 17 00:00:00 2001 From: elkoled Date: Wed, 4 Feb 2026 15:37:58 -0800 Subject: [PATCH 16/16] fork test --- .github/workflows/tests.yaml | 6 +++--- selfdrive/test/process_replay/test_processes.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 03727776bd4a9a..99aae4164eeabd 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -214,14 +214,14 @@ jobs: name: diff_report_${{ github.event.number }} path: selfdrive/test/process_replay/diff_report.txt - name: Checkout ci-artifacts - if: github.repository == 'commaai/openpilot' && github.ref == 'refs/heads/master' + if: github.repository == 'elkoled/openpilot-stock' && github.ref == 'refs/heads/master' uses: actions/checkout@v4 with: - repository: commaai/ci-artifacts + repository: elkoled/ci-artifacts ssh-key: ${{ secrets.CI_ARTIFACTS_DEPLOY_KEY }} path: ${{ github.workspace }}/ci-artifacts - name: Push refs - if: github.repository == 'commaai/openpilot' && github.ref == 'refs/heads/master' + if: github.repository == 'elkoled/openpilot-stock' && github.ref == 'refs/heads/master' working-directory: ${{ github.workspace }}/ci-artifacts run: | git checkout --orphan process-replay diff --git a/selfdrive/test/process_replay/test_processes.py b/selfdrive/test/process_replay/test_processes.py index 7c3318b6a11a44..ce9d84234a6249 100755 --- a/selfdrive/test/process_replay/test_processes.py +++ b/selfdrive/test/process_replay/test_processes.py @@ -67,7 +67,7 @@ # dashcamOnly makes don't need to be tested until a full port is done excluded_interfaces = ["mock", "body", "psa"] -BASE_URL = "https://raw.githubusercontent.com/commaai/ci-artifacts/refs/heads/process-replay/" +BASE_URL = "https://raw.githubusercontent.com/elkoled/ci-artifacts/refs/heads/process-replay/" REF_COMMIT_FN = os.path.join(PROC_REPLAY_DIR, "ref_commit") EXCLUDED_PROCS = {"modeld", "dmonitoringmodeld"}