Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
e6ea7d4
safety replay on the rack
elkoled Nov 14, 2026
7dbc16b
8 replay jobs, alfred allows 20 jobs
elkoled Nov 14, 2026
d7df6e8
Merge remote-tracking branch 'origin/master' into safety-replay
elkoled Nov 16, 2026
885f133
bump opendbc
elkoled Nov 16, 2026
fb355de
clearer comments
elkoled Nov 16, 2026
f610e65
safety replay: retry route download
elkoled Nov 16, 2026
81b1f60
safety replay: shorter job names
elkoled Nov 16, 2026
2b0b680
Merge branch 'master' into safety-replay
elkoled Nov 16, 2026
3f206b8
bump opendbc and panda
elkoled Nov 17, 2026
a9b989d
safety replay: prepare the next route while one replays, job timeouts
elkoled Nov 17, 2026
3a03bc4
safety replay: skip inputs master passed, compare safety with the pre…
elkoled Nov 17, 2026
55bdaab
safety diff: quote expansions
elkoled Nov 17, 2026
cb8803c
safety replay: compare safety in openpilot, opendbc master
elkoled Nov 17, 2026
0b889ff
safety replay: review fixes, handle new cars and older opendbc
elkoled Nov 17, 2026
4f8a7e7
safety replay: allow for the UP's CAN lanes starting a few frames apart
elkoled Nov 17, 2026
f93f4be
safety replay: diff_safety.py reports every change in the diff report…
elkoled Nov 17, 2026
3af1361
diff_safety: boolean waveforms through car_diff.format_diff
elkoled Nov 17, 2026
aa494e5
diff report: share the report layout with the safety replay
elkoled Nov 17, 2026
2e99b6e
safety diff report: own paragraph in the comment, fit the comment size
elkoled Nov 17, 2026
4e78808
diff_safety: compare the whole safety state struct, a fresh libsafety…
elkoled Nov 17, 2026
503f19b
safety replay: derive constants from their sources
elkoled Nov 17, 2026
9080819
safety replay CI: skip when opendbc is unchanged, no build, one libsa…
elkoled Nov 17, 2026
c579e09
diff_safety: full report only in the artifact
elkoled Nov 17, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 31 additions & 2 deletions .github/workflows/diff_report.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -48,11 +48,40 @@ jobs:
run-id: ${{ steps.tests-run.outputs.run-id }}
name: diff_report_${{ github.event.number }}
path: .
- name: Wait for safety replay
if: steps.wait.outcome == 'success'
id: wait-safety
continue-on-error: true
uses: lewagon/wait-on-check-action@1d57e2c51a58d812d2765e036a028b6bdb5a6154
with:
ref: ${{ github.event.pull_request.head.sha }}
check-name: safety replay
repo-token: ${{ secrets.GITHUB_TOKEN }}
allowed-conclusions: success,failure
wait-interval: 20
- name: Download safety diff
if: steps.wait-safety.outcome == 'success'
continue-on-error: true
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
repository: ${{ github.repository }}
run-id: ${{ steps.tests-run.outputs.run-id }}
name: diff_safety_${{ github.event.number }}
path: .
# one comment for both reports, the safety one only when this PR changes opendbc
- name: Combine reports
if: steps.wait.outcome == 'success'
run: |
for f in diff_report.txt diff_safety.txt; do
if [ -f "$f" ]; then cat "$f"; printf '\n\n'; fi
done > comment.md
[ -s comment.md ] || rm -f comment.md
- name: Comment on PR
if: steps.wait.outcome == 'success' && hashFiles('diff_report.txt') != ''
if: steps.wait.outcome == 'success' && hashFiles('comment.md') != ''
uses: thollander/actions-comment-pull-request@24bffb9b452ba05a4f3f77933840a6a841d1b32b
with:
file-path: diff_report.txt
file-path: comment.md
comment-tag: diff_report
pr-number: ${{ github.event.number }}
github-token: ${{ secrets.GITHUB_TOKEN }}
40 changes: 40 additions & 0 deletions .github/workflows/tests.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -174,6 +174,46 @@ jobs:
max_attempts: 3
command: cd ${{ github.workspace }}/ci-artifacts && git push origin process-replay --force

safety_replay:
name: safety replay
runs-on: ${{
(github.repository == 'commaai/openpilot') &&
((github.event_name != 'pull_request') ||
(github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
&& fromJSON('["namespace-profile-amd64-8x16"]')
|| fromJSON('["ubuntu-24.04"]') }}
env:
COMMA_CACHE: /tmp/comma_download_cache
steps:
- uses: actions/checkout@v7
with:
# the parent is master, or the previous master commit on a push
fetch-depth: 2
- run: ./tools/op.sh setup
- name: Check for an opendbc change
id: opendbc
run: openpilot/selfdrive/pandad/tests/diff_safety.py check >> "$GITHUB_OUTPUT"
- name: Cache the test routes
if: steps.opendbc.outputs.changed == 'true'
uses: actions/cache@v6
with:
path: /tmp/comma_download_cache
key: safety-replay-${{ hashFiles('opendbc_repo/opendbc/car/tests/routes.py') }}
restore-keys: safety-replay-
- name: Compare safety with the previous opendbc
if: steps.opendbc.outputs.changed == 'true'
timeout-minutes: 20
run: openpilot/selfdrive/pandad/tests/diff_safety.py
- name: Upload diff report
if: always() && github.event_name == 'pull_request'
uses: actions/upload-artifact@v7
with:
name: diff_safety_${{ github.event.number }}
path: |
diff_safety.txt
diff_safety_full.txt
if-no-files-found: ignore

simulator_driving:
name: simulator driving
runs-on: ${{
Expand Down
53 changes: 53 additions & 0 deletions alfred.toml
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,10 @@ for p in msgq_repo/msgq opendbc_repo/opendbc rednose_repo/rednose teleoprtc_repo
tools/op.sh build
"""

[steps.safety-replay]
name = "Safety replay"
run = "./openpilot/selfdrive/pandad/tests/safety_replay.py"

[steps.flash-chestnut]
name = "Flash Chestnut"
run = "sudo python3 openpilot/system/hardware/chestnut/flash.py"
Expand Down Expand Up @@ -117,6 +121,55 @@ uses = "build"
name = "Sensord"
run = "./openpilot/system/sensord/tests/test_sensord.py"

# each job replays an eighth of the routes, unless master passed the same inputs in the last day
[jobs.safety_0]
runs-on = "up-can"
timeout-minutes = 25
env = { NUM_JOBS = "8", JOB_ID = "0" }
steps = [{ uses = "build" }, { uses = "safety-replay" }]

[jobs.safety_1]
runs-on = "up-can"
timeout-minutes = 25
env = { NUM_JOBS = "8", JOB_ID = "1" }
steps = [{ uses = "build" }, { uses = "safety-replay" }]

[jobs.safety_2]
runs-on = "up-can"
timeout-minutes = 25
env = { NUM_JOBS = "8", JOB_ID = "2" }
steps = [{ uses = "build" }, { uses = "safety-replay" }]

[jobs.safety_3]
runs-on = "up-can"
timeout-minutes = 25
env = { NUM_JOBS = "8", JOB_ID = "3" }
steps = [{ uses = "build" }, { uses = "safety-replay" }]

[jobs.safety_4]
runs-on = "up-can"
timeout-minutes = 25
env = { NUM_JOBS = "8", JOB_ID = "4" }
steps = [{ uses = "build" }, { uses = "safety-replay" }]

[jobs.safety_5]
runs-on = "up-can"
timeout-minutes = 25
env = { NUM_JOBS = "8", JOB_ID = "5" }
steps = [{ uses = "build" }, { uses = "safety-replay" }]

[jobs.safety_6]
runs-on = "up-can"
timeout-minutes = 25
env = { NUM_JOBS = "8", JOB_ID = "6" }
steps = [{ uses = "build" }, { uses = "safety-replay" }]

[jobs.safety_7]
runs-on = "up-can"
timeout-minutes = 25
env = { NUM_JOBS = "8", JOB_ID = "7" }
steps = [{ uses = "build" }, { uses = "safety-replay" }]

# a comma four with a Chestnut on its aux port, which runs the big model
[jobs.chestnut]
runs-on = "chestnut"
Expand Down
2 changes: 1 addition & 1 deletion opendbc_repo
Submodule opendbc_repo updated 95 files
+3 −2 opendbc/can/packer.py
+3 −2 opendbc/can/parser.py
+9 −8 opendbc/car/__init__.py
+5 −17 opendbc/car/chrysler/radar_interface.py
+8 −9 opendbc/car/common/pid.py
+3 −2 opendbc/car/docs_definitions.py
+2 −1 opendbc/car/ford/fingerprints.py
+11 −23 opendbc/car/ford/radar_interface.py
+5 −5 opendbc/car/ford/tests/test_ford.py
+1 −1 opendbc/car/ford/values.py
+1 −1 opendbc/car/fw_versions.py
+7 −20 opendbc/car/gm/radar_interface.py
+5 −26 opendbc/car/honda/radar_interface.py
+4 −25 opendbc/car/hyundai/radar_interface.py
+3 −3 opendbc/car/hyundai/tests/test_hyundai.py
+21 −6 opendbc/car/interfaces.py
+1 −1 opendbc/car/mock/carstate.py
+4 −25 opendbc/car/rivian/radar_interface.py
+2 −2 opendbc/car/rivian/tests/test_rivian.py
+2 −0 opendbc/car/rivian/values.py
+6 −27 opendbc/car/tesla/radar_interface.py
+4 −4 opendbc/car/tests/car_diff.py
+2 −2 opendbc/car/tests/test_can_fingerprint.py
+5 −6 opendbc/car/tests/test_car_interfaces.py
+4 −4 opendbc/car/tests/test_fw_fingerprint.py
+6 −6 opendbc/car/tests/test_models.py
+1 −1 opendbc/car/tests/test_platform_configs.py
+6 −26 opendbc/car/toyota/radar_interface.py
+4 −4 opendbc/car/toyota/tests/test_toyota.py
+1 −1 opendbc/car/toyota/values.py
+14 −18 opendbc/car/uds.py
+1 −1 opendbc/car/volkswagen/carcontroller.py
+4 −15 opendbc/car/volkswagen/radar_interface.py
+3 −3 opendbc/car/volkswagen/tests/test_volkswagen.py
+0 −1 opendbc/safety/__init__.py
+1 −1 opendbc/safety/can.h
+32 −21 opendbc/safety/declarations.h
+43 −26 opendbc/safety/helpers.h
+30 −17 opendbc/safety/ignition.h
+105 −103 opendbc/safety/lateral.h
+8 −4 opendbc/safety/longitudinal.h
+4 −1 opendbc/safety/modes/body.h
+35 −36 opendbc/safety/modes/byd.h
+11 −9 opendbc/safety/modes/chrysler.h
+7 −4 opendbc/safety/modes/chrysler_common.h
+9 −5 opendbc/safety/modes/chrysler_cusw.h
+8 −19 opendbc/safety/modes/defaults.h
+7 −5 opendbc/safety/modes/elm327.h
+41 −38 opendbc/safety/modes/ford.h
+22 −20 opendbc/safety/modes/gm.h
+15 −22 opendbc/safety/modes/honda.h
+29 −29 opendbc/safety/modes/hyundai.h
+18 −18 opendbc/safety/modes/hyundai_canfd.h
+43 −43 opendbc/safety/modes/hyundai_common.h
+11 −8 opendbc/safety/modes/mazda.h
+16 −15 opendbc/safety/modes/mg.h
+26 −19 opendbc/safety/modes/nissan.h
+22 −19 opendbc/safety/modes/psa.h
+26 −23 opendbc/safety/modes/rivian.h
+20 −23 opendbc/safety/modes/subaru.h
+8 −8 opendbc/safety/modes/subaru_preglobal.h
+81 −91 opendbc/safety/modes/tesla.h
+36 −38 opendbc/safety/modes/toyota.h
+33 −15 opendbc/safety/modes/volkswagen_common.h
+27 −40 opendbc/safety/modes/volkswagen_meb.h
+11 −9 opendbc/safety/modes/volkswagen_mlb.h
+17 −31 opendbc/safety/modes/volkswagen_mqb.h
+19 −30 opendbc/safety/modes/volkswagen_pq.h
+102 −86 opendbc/safety/safety.h
+14 −13 opendbc/safety/tests/common.py
+10 −1 opendbc/safety/tests/hyundai_common.py
+24 −11 opendbc/safety/tests/libsafety/libsafety_py.py
+8 −1 opendbc/safety/tests/libsafety/safety.c
+15 −12 opendbc/safety/tests/misra/main.c
+3 −1 opendbc/safety/tests/misra/test_misra.sh
+1 −1 opendbc/safety/tests/misra/test_mutation.py
+499 −572 opendbc/safety/tests/mutation.py
+3 −3 opendbc/safety/tests/test.sh
+7 −2 opendbc/safety/tests/test_byd.py
+59 −0 opendbc/safety/tests/test_defaults.py
+5 −2 opendbc/safety/tests/test_ford.py
+23 −2 opendbc/safety/tests/test_gm.py
+27 −6 opendbc/safety/tests/test_honda.py
+36 −6 opendbc/safety/tests/test_hyundai.py
+27 −6 opendbc/safety/tests/test_hyundai_canfd.py
+7 −0 opendbc/safety/tests/test_mg.py
+13 −3 opendbc/safety/tests/test_tesla.py
+15 −13 opendbc/safety/tests/test_toyota.py
+8 −10 opendbc/safety/tests/test_volkswagen_meb.py
+8 −0 opendbc/safety/tests/test_volkswagen_mlb.py
+13 −7 opendbc/safety/tests/test_volkswagen_mqb.py
+7 −0 opendbc/safety/tests/test_volkswagen_pq.py
+1 −1 opendbc/testing.py
+12 −27 pyproject.toml
+138 −134 uv.lock
Loading
Loading