From 03604bae3f8783d278eca2cc461898d2813ea4fb Mon Sep 17 00:00:00 2001 From: Andrii Shylenko <14119286+w1ne@users.noreply.github.com> Date: Sun, 16 Aug 2026 18:18:00 +0200 Subject: [PATCH 1/2] feat(hardware): diff the twin against a real desk board Same firmware artifact, two independent targets: the LabWired digital twin and a physical desk board. `hardware diff` compares the two authenticated evidence bundles and publishes the disagreement as a first-class result. Each side is summarized from its own bundle only, so there is no code path that upgrades a hardware green to a twin green or the reverse. A twin bundle that records `hardware_observed` is rejected as invalid. Both sides must bind to the same artifact digest. A missing probe or a missing desk bundle yields `desk-unavailable` (exit 4), never a silent pass. Verdicts bind to exit codes: agree 0, invalid 2, disagree 3, desk-unavailable 4, twin-unavailable 5. A failed diff invocation returns 2 so it can never borrow the disagree code. Tests run entirely from recorded evidence bundles under fixtures/twin-desk-diff, so the differential is provable with no board attached. --- .gitignore | 2 + bin/labwired-agent | 4 + docs/USAGE.md | 34 ++ .../twin-desk-diff/agree/desk.receipt.json | 7 + .../twin-desk-diff/agree/desk/.owner.json | 31 ++ .../observations/heartbeat.desk.capture.log | 2 + .../desk/observations/heartbeat/result.json | 28 ++ .../observations/led-blink.desk.capture.log | 2 + .../desk/observations/led-blink/result.json | 28 ++ .../twin-desk-diff/agree/desk/result.json | 4 + .../agree/desk/stages/build/result.json | 5 + .../agree/desk/stages/flash/result.json | 5 + .../twin-desk-diff/agree/twin.receipt.json | 7 + .../twin-desk-diff/agree/twin/.owner.json | 31 ++ .../observations/heartbeat.twin.capture.log | 2 + .../twin/observations/heartbeat/result.json | 28 ++ .../observations/led-blink.twin.capture.log | 2 + .../twin/observations/led-blink/result.json | 28 ++ .../twin-desk-diff/agree/twin/result.json | 4 + .../agree/twin/stages/build/result.json | 5 + .../agree/twin/stages/twin/result.json | 5 + .../contaminated/twin.receipt.json | 7 + .../contaminated/twin/.owner.json | 31 ++ .../observations/heartbeat.twin.capture.log | 2 + .../twin/observations/heartbeat/result.json | 28 ++ .../observations/led-blink.twin.capture.log | 2 + .../twin/observations/led-blink/result.json | 28 ++ .../contaminated/twin/result.json | 4 + .../twin/stages/build/result.json | 5 + .../contaminated/twin/stages/twin/result.json | 5 + .../twin-desk-diff/disagree/desk.receipt.json | 7 + .../twin-desk-diff/disagree/desk/.owner.json | 31 ++ .../observations/heartbeat.desk.capture.log | 2 + .../desk/observations/heartbeat/result.json | 28 ++ .../observations/led-blink.desk.capture.log | 3 + .../desk/observations/led-blink/result.json | 19 + .../twin-desk-diff/disagree/desk/result.json | 11 + .../disagree/desk/stages/build/result.json | 5 + .../disagree/desk/stages/flash/result.json | 5 + fixtures/twin-desk-diff/firmware-a.bin | 1 + fixtures/twin-desk-diff/firmware-b.bin | 1 + .../twin-desk-diff/mismatch/desk.receipt.json | 7 + .../twin-desk-diff/mismatch/desk/.owner.json | 31 ++ .../observations/heartbeat.desk.capture.log | 2 + .../desk/observations/heartbeat/result.json | 28 ++ .../observations/led-blink.desk.capture.log | 2 + .../desk/observations/led-blink/result.json | 28 ++ .../twin-desk-diff/mismatch/desk/result.json | 4 + .../mismatch/desk/stages/build/result.json | 5 + .../mismatch/desk/stages/flash/result.json | 5 + .../twin-desk-diff/no-probe/desk.receipt.json | 7 + .../twin-desk-diff/no-probe/desk/.owner.json | 31 ++ .../observations/heartbeat.desk.capture.log | 3 + .../desk/observations/heartbeat/result.json | 19 + .../observations/led-blink.desk.capture.log | 3 + .../desk/observations/led-blink/result.json | 19 + .../twin-desk-diff/no-probe/desk/result.json | 17 + .../no-probe/desk/stages/build/result.json | 5 + .../no-probe/desk/stages/flash/result.json | 5 + lib/hardware/differential.mjs | 324 ++++++++++++++++++ package.json | 2 +- scripts/hardware-runner.mjs | 59 +++- skills/desk-hw/SKILL.md | 16 + tests/hardware-differential.test.mjs | 242 +++++++++++++ tests/hardware-public-docs.sh | 10 + tests/helpers/make-twin-desk-fixtures.mjs | 143 ++++++++ 66 files changed, 1503 insertions(+), 3 deletions(-) create mode 100644 fixtures/twin-desk-diff/agree/desk.receipt.json create mode 100644 fixtures/twin-desk-diff/agree/desk/.owner.json create mode 100644 fixtures/twin-desk-diff/agree/desk/observations/heartbeat.desk.capture.log create mode 100644 fixtures/twin-desk-diff/agree/desk/observations/heartbeat/result.json create mode 100644 fixtures/twin-desk-diff/agree/desk/observations/led-blink.desk.capture.log create mode 100644 fixtures/twin-desk-diff/agree/desk/observations/led-blink/result.json create mode 100644 fixtures/twin-desk-diff/agree/desk/result.json create mode 100644 fixtures/twin-desk-diff/agree/desk/stages/build/result.json create mode 100644 fixtures/twin-desk-diff/agree/desk/stages/flash/result.json create mode 100644 fixtures/twin-desk-diff/agree/twin.receipt.json create mode 100644 fixtures/twin-desk-diff/agree/twin/.owner.json create mode 100644 fixtures/twin-desk-diff/agree/twin/observations/heartbeat.twin.capture.log create mode 100644 fixtures/twin-desk-diff/agree/twin/observations/heartbeat/result.json create mode 100644 fixtures/twin-desk-diff/agree/twin/observations/led-blink.twin.capture.log create mode 100644 fixtures/twin-desk-diff/agree/twin/observations/led-blink/result.json create mode 100644 fixtures/twin-desk-diff/agree/twin/result.json create mode 100644 fixtures/twin-desk-diff/agree/twin/stages/build/result.json create mode 100644 fixtures/twin-desk-diff/agree/twin/stages/twin/result.json create mode 100644 fixtures/twin-desk-diff/contaminated/twin.receipt.json create mode 100644 fixtures/twin-desk-diff/contaminated/twin/.owner.json create mode 100644 fixtures/twin-desk-diff/contaminated/twin/observations/heartbeat.twin.capture.log create mode 100644 fixtures/twin-desk-diff/contaminated/twin/observations/heartbeat/result.json create mode 100644 fixtures/twin-desk-diff/contaminated/twin/observations/led-blink.twin.capture.log create mode 100644 fixtures/twin-desk-diff/contaminated/twin/observations/led-blink/result.json create mode 100644 fixtures/twin-desk-diff/contaminated/twin/result.json create mode 100644 fixtures/twin-desk-diff/contaminated/twin/stages/build/result.json create mode 100644 fixtures/twin-desk-diff/contaminated/twin/stages/twin/result.json create mode 100644 fixtures/twin-desk-diff/disagree/desk.receipt.json create mode 100644 fixtures/twin-desk-diff/disagree/desk/.owner.json create mode 100644 fixtures/twin-desk-diff/disagree/desk/observations/heartbeat.desk.capture.log create mode 100644 fixtures/twin-desk-diff/disagree/desk/observations/heartbeat/result.json create mode 100644 fixtures/twin-desk-diff/disagree/desk/observations/led-blink.desk.capture.log create mode 100644 fixtures/twin-desk-diff/disagree/desk/observations/led-blink/result.json create mode 100644 fixtures/twin-desk-diff/disagree/desk/result.json create mode 100644 fixtures/twin-desk-diff/disagree/desk/stages/build/result.json create mode 100644 fixtures/twin-desk-diff/disagree/desk/stages/flash/result.json create mode 100644 fixtures/twin-desk-diff/firmware-a.bin create mode 100644 fixtures/twin-desk-diff/firmware-b.bin create mode 100644 fixtures/twin-desk-diff/mismatch/desk.receipt.json create mode 100644 fixtures/twin-desk-diff/mismatch/desk/.owner.json create mode 100644 fixtures/twin-desk-diff/mismatch/desk/observations/heartbeat.desk.capture.log create mode 100644 fixtures/twin-desk-diff/mismatch/desk/observations/heartbeat/result.json create mode 100644 fixtures/twin-desk-diff/mismatch/desk/observations/led-blink.desk.capture.log create mode 100644 fixtures/twin-desk-diff/mismatch/desk/observations/led-blink/result.json create mode 100644 fixtures/twin-desk-diff/mismatch/desk/result.json create mode 100644 fixtures/twin-desk-diff/mismatch/desk/stages/build/result.json create mode 100644 fixtures/twin-desk-diff/mismatch/desk/stages/flash/result.json create mode 100644 fixtures/twin-desk-diff/no-probe/desk.receipt.json create mode 100644 fixtures/twin-desk-diff/no-probe/desk/.owner.json create mode 100644 fixtures/twin-desk-diff/no-probe/desk/observations/heartbeat.desk.capture.log create mode 100644 fixtures/twin-desk-diff/no-probe/desk/observations/heartbeat/result.json create mode 100644 fixtures/twin-desk-diff/no-probe/desk/observations/led-blink.desk.capture.log create mode 100644 fixtures/twin-desk-diff/no-probe/desk/observations/led-blink/result.json create mode 100644 fixtures/twin-desk-diff/no-probe/desk/result.json create mode 100644 fixtures/twin-desk-diff/no-probe/desk/stages/build/result.json create mode 100644 fixtures/twin-desk-diff/no-probe/desk/stages/flash/result.json create mode 100644 lib/hardware/differential.mjs create mode 100644 tests/hardware-differential.test.mjs create mode 100644 tests/helpers/make-twin-desk-fixtures.mjs diff --git a/.gitignore b/.gitignore index 9d6f193..d7a5c3a 100644 --- a/.gitignore +++ b/.gitignore @@ -44,6 +44,8 @@ extensions/labwired-vscode/share/tools.json fixtures/**/.pio/ fixtures/**/evidence/ fixtures/coverage/smoke/ +# Recorded twin-vs-desk differential fixtures (stand-in artifacts, tracked on purpose) +!fixtures/twin-desk-diff/*.bin workspaces/**/.pio/ workspaces/**/evidence/ share/smoke/.dead-token diff --git a/bin/labwired-agent b/bin/labwired-agent index f3a256f..bfbce30 100755 --- a/bin/labwired-agent +++ b/bin/labwired-agent @@ -916,6 +916,10 @@ Usage: labwired agent compose … Assemble plot elements (uart | capture) labwired agent hardware plan --profile FILE --out DIR labwired agent hardware run --profile FILE --out DIR --confirm DIGEST + labwired agent hardware diff --artifact FILE --twin-evidence DIR --twin-receipt SHA256 + [--desk-evidence DIR --desk-receipt SHA256] [--out FILE] + Same firmware, twin vs desk board; exit 0 agree, 3 disagree, + 4 desk-unavailable, 5 twin-unavailable, 2 invalid labwired agent install-deps Install sim + probe-rs into prefix labwired agent package … Portable prefix: info | path | env | uninstall labwired agent version Version diff --git a/docs/USAGE.md b/docs/USAGE.md index c5418d4..b37ab45 100644 --- a/docs/USAGE.md +++ b/docs/USAGE.md @@ -133,3 +133,37 @@ results and execution or evidence `FAIL` results exit `3`. Planning never creates the evidence directory or builds, flashes, or opens an instrument. Runs lock the explicit target, probe, port, and analyzer identities so two sessions cannot control the same lab resource. + +## Differential: the same firmware on the twin and on a desk board + +One firmware artifact, two independent targets. Run the twin lane and the +physical lane as ordinary `hardware run` invocations against their own profiles, +then compare the two authenticated evidence bundles: + +```bash +labwired agent hardware diff --artifact build/firmware.bin \ + --twin-evidence .labwired/evidence-twin --twin-receipt <64-character-twin-receipt> \ + --desk-evidence .labwired/evidence-desk --desk-receipt <64-character-desk-receipt> \ + --out .labwired/twin-desk-diff.json +``` + +Each `--*-receipt` is the out-of-bundle `manifestSha256` that `hardware run` +returned for that bundle. A bundle supplied without its receipt is never +accepted as evidence for its side. + +`hardware diff` emits one structured JSON verdict and binds it to exit codes: + +| Verdict | Exit | Meaning | +|---|---|---| +| `agree` | `0` | Both targets decided the same behaviors the same way. | +| `disagree` | `3` | The twin and the board decided a shared behavior differently. This is a first-class published result, not an error. | +| `desk-unavailable` | `4` | No desk bundle, or the desk bundle recorded no physical evidence (for example no probe was detected). Never a silent pass. | +| `twin-unavailable` | `5` | No twin bundle, or the twin bundle decided nothing. | +| `invalid` | `2` | The comparison was refused: unequal artifact digests, a twin bundle claiming `hardware_observed`, no shared behavior, or a CLI usage error. | + +Each side publishes only its own grade. The twin side can reach +`model_verified` (from `model_observed` or `surrogate_model_observed`); the desk +side can reach `hardware_observed`. `hardware diff` has no code path that copies +a level or a pass from one side to the other, so a hardware green is never +upgraded to a twin green or the reverse. Both sides must be bound to the same +artifact digest; a mismatch is `invalid`, never agreement. diff --git a/fixtures/twin-desk-diff/agree/desk.receipt.json b/fixtures/twin-desk-diff/agree/desk.receipt.json new file mode 100644 index 0000000..f238b11 --- /dev/null +++ b/fixtures/twin-desk-diff/agree/desk.receipt.json @@ -0,0 +1,7 @@ +{ + "bundle": "agree/desk", + "side": "desk", + "artifactSha256": "1bd0221812559331be433e5b1e002945ebba25b6503b1fafc85adc4cd1555fcf", + "manifestSha256": "5b627cf5eb053314f3261cb90fcba4dbcb2be301583a58970d7262d4e6d04acb", + "result": "PASS" +} diff --git a/fixtures/twin-desk-diff/agree/desk/.owner.json b/fixtures/twin-desk-diff/agree/desk/.owner.json new file mode 100644 index 0000000..db1c665 --- /dev/null +++ b/fixtures/twin-desk-diff/agree/desk/.owner.json @@ -0,0 +1,31 @@ +{ + "schema": 1, + "owner": "29ce09f1-3ffb-4c9b-b9c4-626b5823fd24", + "createdAt": "2026-08-16T16:11:24.398Z", + "targetIdentity": { + "id": "desk-c3", + "chip": "esp32c3" + }, + "behaviors": [ + { + "id": "heartbeat", + "provider": "serial", + "requiredLevel": "hardware_observed" + }, + { + "id": "led-blink", + "provider": "logic-csv", + "requiredLevel": "hardware_observed" + } + ], + "stages": [ + { + "id": "build", + "provider": "platformio" + }, + { + "id": "flash", + "provider": "probe-rs" + } + ] +} diff --git a/fixtures/twin-desk-diff/agree/desk/observations/heartbeat.desk.capture.log b/fixtures/twin-desk-diff/agree/desk/observations/heartbeat.desk.capture.log new file mode 100644 index 0000000..dff2af5 --- /dev/null +++ b/fixtures/twin-desk-diff/agree/desk/observations/heartbeat.desk.capture.log @@ -0,0 +1,2 @@ +desk capture for heartbeat +level=hardware_observed diff --git a/fixtures/twin-desk-diff/agree/desk/observations/heartbeat/result.json b/fixtures/twin-desk-diff/agree/desk/observations/heartbeat/result.json new file mode 100644 index 0000000..adfdcaa --- /dev/null +++ b/fixtures/twin-desk-diff/agree/desk/observations/heartbeat/result.json @@ -0,0 +1,28 @@ +{ + "behaviorId": "heartbeat", + "provider": "serial", + "level": "hardware_observed", + "artifactSha256": "1bd0221812559331be433e5b1e002945ebba25b6503b1fafc85adc4cd1555fcf", + "targetIdentity": { + "id": "desk-c3", + "chip": "esp32c3" + }, + "startedAt": "2026-08-16T16:11:24.288Z", + "endedAt": "2026-08-16T16:11:28.288Z", + "toolVersion": "probe-rs recorded fixture", + "rawEvidenceRefs": [ + "observations/heartbeat.desk.capture.log" + ], + "diagnostics": { + "detail": "desk observed heartbeat" + }, + "flashedArtifactSha256": "1bd0221812559331be433e5b1e002945ebba25b6503b1fafc85adc4cd1555fcf", + "rawEvidence": [ + { + "path": "observations/heartbeat.desk.capture.log", + "sha256": "5aba5981cb190ef4cd9d90f657261630d61d8845bcbc6885b24a4703c45f4bc6", + "size": 51 + } + ], + "requiredLevel": "hardware_observed" +} diff --git a/fixtures/twin-desk-diff/agree/desk/observations/led-blink.desk.capture.log b/fixtures/twin-desk-diff/agree/desk/observations/led-blink.desk.capture.log new file mode 100644 index 0000000..955cdb4 --- /dev/null +++ b/fixtures/twin-desk-diff/agree/desk/observations/led-blink.desk.capture.log @@ -0,0 +1,2 @@ +desk capture for led-blink +level=hardware_observed diff --git a/fixtures/twin-desk-diff/agree/desk/observations/led-blink/result.json b/fixtures/twin-desk-diff/agree/desk/observations/led-blink/result.json new file mode 100644 index 0000000..5178e91 --- /dev/null +++ b/fixtures/twin-desk-diff/agree/desk/observations/led-blink/result.json @@ -0,0 +1,28 @@ +{ + "behaviorId": "led-blink", + "provider": "logic-csv", + "level": "hardware_observed", + "artifactSha256": "1bd0221812559331be433e5b1e002945ebba25b6503b1fafc85adc4cd1555fcf", + "targetIdentity": { + "id": "desk-c3", + "chip": "esp32c3" + }, + "startedAt": "2026-08-16T16:11:24.288Z", + "endedAt": "2026-08-16T16:11:28.288Z", + "toolVersion": "probe-rs recorded fixture", + "rawEvidenceRefs": [ + "observations/led-blink.desk.capture.log" + ], + "diagnostics": { + "detail": "desk observed led-blink" + }, + "flashedArtifactSha256": "1bd0221812559331be433e5b1e002945ebba25b6503b1fafc85adc4cd1555fcf", + "rawEvidence": [ + { + "path": "observations/led-blink.desk.capture.log", + "sha256": "5b3cacc1d608a94df1f8c95d3393e8e8c7c4f2a341da0048f2e1bdc1fbbe8fcf", + "size": 51 + } + ], + "requiredLevel": "hardware_observed" +} diff --git a/fixtures/twin-desk-diff/agree/desk/result.json b/fixtures/twin-desk-diff/agree/desk/result.json new file mode 100644 index 0000000..3426323 --- /dev/null +++ b/fixtures/twin-desk-diff/agree/desk/result.json @@ -0,0 +1,4 @@ +{ + "result": "PASS", + "reasons": [] +} diff --git a/fixtures/twin-desk-diff/agree/desk/stages/build/result.json b/fixtures/twin-desk-diff/agree/desk/stages/build/result.json new file mode 100644 index 0000000..be85c93 --- /dev/null +++ b/fixtures/twin-desk-diff/agree/desk/stages/build/result.json @@ -0,0 +1,5 @@ +{ + "stageId": "build", + "provider": "platformio", + "level": "not-run" +} diff --git a/fixtures/twin-desk-diff/agree/desk/stages/flash/result.json b/fixtures/twin-desk-diff/agree/desk/stages/flash/result.json new file mode 100644 index 0000000..b45d301 --- /dev/null +++ b/fixtures/twin-desk-diff/agree/desk/stages/flash/result.json @@ -0,0 +1,5 @@ +{ + "stageId": "flash", + "provider": "probe-rs", + "level": "not-run" +} diff --git a/fixtures/twin-desk-diff/agree/twin.receipt.json b/fixtures/twin-desk-diff/agree/twin.receipt.json new file mode 100644 index 0000000..9bf3386 --- /dev/null +++ b/fixtures/twin-desk-diff/agree/twin.receipt.json @@ -0,0 +1,7 @@ +{ + "bundle": "agree/twin", + "side": "twin", + "artifactSha256": "1bd0221812559331be433e5b1e002945ebba25b6503b1fafc85adc4cd1555fcf", + "manifestSha256": "5a66eead2d11fd4fa2fca5352dbdbe8560753b7e5e4b291cd54fd17486da8191", + "result": "PASS" +} diff --git a/fixtures/twin-desk-diff/agree/twin/.owner.json b/fixtures/twin-desk-diff/agree/twin/.owner.json new file mode 100644 index 0000000..29dc5ce --- /dev/null +++ b/fixtures/twin-desk-diff/agree/twin/.owner.json @@ -0,0 +1,31 @@ +{ + "schema": 1, + "owner": "62bb0ca3-5b92-440d-bf02-323553bac607", + "createdAt": "2026-08-16T16:11:24.299Z", + "targetIdentity": { + "id": "desk-c3", + "chip": "esp32c3" + }, + "behaviors": [ + { + "id": "heartbeat", + "provider": "serial", + "requiredLevel": "model_observed" + }, + { + "id": "led-blink", + "provider": "logic-csv", + "requiredLevel": "model_observed" + } + ], + "stages": [ + { + "id": "build", + "provider": "platformio" + }, + { + "id": "twin", + "provider": "labwired-sim" + } + ] +} diff --git a/fixtures/twin-desk-diff/agree/twin/observations/heartbeat.twin.capture.log b/fixtures/twin-desk-diff/agree/twin/observations/heartbeat.twin.capture.log new file mode 100644 index 0000000..8f384d4 --- /dev/null +++ b/fixtures/twin-desk-diff/agree/twin/observations/heartbeat.twin.capture.log @@ -0,0 +1,2 @@ +twin capture for heartbeat +level=model_observed diff --git a/fixtures/twin-desk-diff/agree/twin/observations/heartbeat/result.json b/fixtures/twin-desk-diff/agree/twin/observations/heartbeat/result.json new file mode 100644 index 0000000..248989a --- /dev/null +++ b/fixtures/twin-desk-diff/agree/twin/observations/heartbeat/result.json @@ -0,0 +1,28 @@ +{ + "behaviorId": "heartbeat", + "provider": "serial", + "level": "model_observed", + "artifactSha256": "1bd0221812559331be433e5b1e002945ebba25b6503b1fafc85adc4cd1555fcf", + "targetIdentity": { + "id": "desk-c3", + "chip": "esp32c3" + }, + "startedAt": "2026-08-16T16:11:24.288Z", + "endedAt": "2026-08-16T16:11:28.288Z", + "toolVersion": "labwired-sim recorded fixture", + "rawEvidenceRefs": [ + "observations/heartbeat.twin.capture.log" + ], + "diagnostics": { + "detail": "twin observed heartbeat" + }, + "nativeArtifactSha256": "1bd0221812559331be433e5b1e002945ebba25b6503b1fafc85adc4cd1555fcf", + "rawEvidence": [ + { + "path": "observations/heartbeat.twin.capture.log", + "sha256": "075b5e80a20b12fb8149f6d12d5fbbaa93ce2860f143dda0cb806d2f958de887", + "size": 48 + } + ], + "requiredLevel": "model_observed" +} diff --git a/fixtures/twin-desk-diff/agree/twin/observations/led-blink.twin.capture.log b/fixtures/twin-desk-diff/agree/twin/observations/led-blink.twin.capture.log new file mode 100644 index 0000000..542dd25 --- /dev/null +++ b/fixtures/twin-desk-diff/agree/twin/observations/led-blink.twin.capture.log @@ -0,0 +1,2 @@ +twin capture for led-blink +level=model_observed diff --git a/fixtures/twin-desk-diff/agree/twin/observations/led-blink/result.json b/fixtures/twin-desk-diff/agree/twin/observations/led-blink/result.json new file mode 100644 index 0000000..2acad7a --- /dev/null +++ b/fixtures/twin-desk-diff/agree/twin/observations/led-blink/result.json @@ -0,0 +1,28 @@ +{ + "behaviorId": "led-blink", + "provider": "logic-csv", + "level": "model_observed", + "artifactSha256": "1bd0221812559331be433e5b1e002945ebba25b6503b1fafc85adc4cd1555fcf", + "targetIdentity": { + "id": "desk-c3", + "chip": "esp32c3" + }, + "startedAt": "2026-08-16T16:11:24.288Z", + "endedAt": "2026-08-16T16:11:28.288Z", + "toolVersion": "labwired-sim recorded fixture", + "rawEvidenceRefs": [ + "observations/led-blink.twin.capture.log" + ], + "diagnostics": { + "detail": "twin observed led-blink" + }, + "nativeArtifactSha256": "1bd0221812559331be433e5b1e002945ebba25b6503b1fafc85adc4cd1555fcf", + "rawEvidence": [ + { + "path": "observations/led-blink.twin.capture.log", + "sha256": "706f06775ea2096934a20e3349cf37f8617b6108171ae90f20265282a7021703", + "size": 48 + } + ], + "requiredLevel": "model_observed" +} diff --git a/fixtures/twin-desk-diff/agree/twin/result.json b/fixtures/twin-desk-diff/agree/twin/result.json new file mode 100644 index 0000000..3426323 --- /dev/null +++ b/fixtures/twin-desk-diff/agree/twin/result.json @@ -0,0 +1,4 @@ +{ + "result": "PASS", + "reasons": [] +} diff --git a/fixtures/twin-desk-diff/agree/twin/stages/build/result.json b/fixtures/twin-desk-diff/agree/twin/stages/build/result.json new file mode 100644 index 0000000..be85c93 --- /dev/null +++ b/fixtures/twin-desk-diff/agree/twin/stages/build/result.json @@ -0,0 +1,5 @@ +{ + "stageId": "build", + "provider": "platformio", + "level": "not-run" +} diff --git a/fixtures/twin-desk-diff/agree/twin/stages/twin/result.json b/fixtures/twin-desk-diff/agree/twin/stages/twin/result.json new file mode 100644 index 0000000..3aae568 --- /dev/null +++ b/fixtures/twin-desk-diff/agree/twin/stages/twin/result.json @@ -0,0 +1,5 @@ +{ + "stageId": "twin", + "provider": "labwired-sim", + "level": "not-run" +} diff --git a/fixtures/twin-desk-diff/contaminated/twin.receipt.json b/fixtures/twin-desk-diff/contaminated/twin.receipt.json new file mode 100644 index 0000000..581dbb8 --- /dev/null +++ b/fixtures/twin-desk-diff/contaminated/twin.receipt.json @@ -0,0 +1,7 @@ +{ + "bundle": "contaminated/twin", + "side": "twin", + "artifactSha256": "1bd0221812559331be433e5b1e002945ebba25b6503b1fafc85adc4cd1555fcf", + "manifestSha256": "fd5ebe8fec4af5cb8f487fbe8a3f49a074ea9d27758896e516e7ad9a9f4ec992", + "result": "PASS" +} diff --git a/fixtures/twin-desk-diff/contaminated/twin/.owner.json b/fixtures/twin-desk-diff/contaminated/twin/.owner.json new file mode 100644 index 0000000..ebd0e9e --- /dev/null +++ b/fixtures/twin-desk-diff/contaminated/twin/.owner.json @@ -0,0 +1,31 @@ +{ + "schema": 1, + "owner": "da33f05a-b86a-4932-abd3-b1d0535f4784", + "createdAt": "2026-08-16T16:11:24.591Z", + "targetIdentity": { + "id": "desk-c3", + "chip": "esp32c3" + }, + "behaviors": [ + { + "id": "heartbeat", + "provider": "serial", + "requiredLevel": "model_observed" + }, + { + "id": "led-blink", + "provider": "logic-csv", + "requiredLevel": "model_observed" + } + ], + "stages": [ + { + "id": "build", + "provider": "platformio" + }, + { + "id": "twin", + "provider": "labwired-sim" + } + ] +} diff --git a/fixtures/twin-desk-diff/contaminated/twin/observations/heartbeat.twin.capture.log b/fixtures/twin-desk-diff/contaminated/twin/observations/heartbeat.twin.capture.log new file mode 100644 index 0000000..508cab7 --- /dev/null +++ b/fixtures/twin-desk-diff/contaminated/twin/observations/heartbeat.twin.capture.log @@ -0,0 +1,2 @@ +twin capture for heartbeat +level=hardware_observed diff --git a/fixtures/twin-desk-diff/contaminated/twin/observations/heartbeat/result.json b/fixtures/twin-desk-diff/contaminated/twin/observations/heartbeat/result.json new file mode 100644 index 0000000..109c572 --- /dev/null +++ b/fixtures/twin-desk-diff/contaminated/twin/observations/heartbeat/result.json @@ -0,0 +1,28 @@ +{ + "behaviorId": "heartbeat", + "provider": "serial", + "level": "hardware_observed", + "artifactSha256": "1bd0221812559331be433e5b1e002945ebba25b6503b1fafc85adc4cd1555fcf", + "targetIdentity": { + "id": "desk-c3", + "chip": "esp32c3" + }, + "startedAt": "2026-08-16T16:11:24.288Z", + "endedAt": "2026-08-16T16:11:28.288Z", + "toolVersion": "labwired-sim recorded fixture", + "rawEvidenceRefs": [ + "observations/heartbeat.twin.capture.log" + ], + "diagnostics": { + "detail": "twin observed heartbeat" + }, + "flashedArtifactSha256": "1bd0221812559331be433e5b1e002945ebba25b6503b1fafc85adc4cd1555fcf", + "rawEvidence": [ + { + "path": "observations/heartbeat.twin.capture.log", + "sha256": "5ac80c869fdcde2df3dee6fd3037bd3b01588c9036339e8647c5b90ce98178d6", + "size": 51 + } + ], + "requiredLevel": "model_observed" +} diff --git a/fixtures/twin-desk-diff/contaminated/twin/observations/led-blink.twin.capture.log b/fixtures/twin-desk-diff/contaminated/twin/observations/led-blink.twin.capture.log new file mode 100644 index 0000000..542dd25 --- /dev/null +++ b/fixtures/twin-desk-diff/contaminated/twin/observations/led-blink.twin.capture.log @@ -0,0 +1,2 @@ +twin capture for led-blink +level=model_observed diff --git a/fixtures/twin-desk-diff/contaminated/twin/observations/led-blink/result.json b/fixtures/twin-desk-diff/contaminated/twin/observations/led-blink/result.json new file mode 100644 index 0000000..2acad7a --- /dev/null +++ b/fixtures/twin-desk-diff/contaminated/twin/observations/led-blink/result.json @@ -0,0 +1,28 @@ +{ + "behaviorId": "led-blink", + "provider": "logic-csv", + "level": "model_observed", + "artifactSha256": "1bd0221812559331be433e5b1e002945ebba25b6503b1fafc85adc4cd1555fcf", + "targetIdentity": { + "id": "desk-c3", + "chip": "esp32c3" + }, + "startedAt": "2026-08-16T16:11:24.288Z", + "endedAt": "2026-08-16T16:11:28.288Z", + "toolVersion": "labwired-sim recorded fixture", + "rawEvidenceRefs": [ + "observations/led-blink.twin.capture.log" + ], + "diagnostics": { + "detail": "twin observed led-blink" + }, + "nativeArtifactSha256": "1bd0221812559331be433e5b1e002945ebba25b6503b1fafc85adc4cd1555fcf", + "rawEvidence": [ + { + "path": "observations/led-blink.twin.capture.log", + "sha256": "706f06775ea2096934a20e3349cf37f8617b6108171ae90f20265282a7021703", + "size": 48 + } + ], + "requiredLevel": "model_observed" +} diff --git a/fixtures/twin-desk-diff/contaminated/twin/result.json b/fixtures/twin-desk-diff/contaminated/twin/result.json new file mode 100644 index 0000000..3426323 --- /dev/null +++ b/fixtures/twin-desk-diff/contaminated/twin/result.json @@ -0,0 +1,4 @@ +{ + "result": "PASS", + "reasons": [] +} diff --git a/fixtures/twin-desk-diff/contaminated/twin/stages/build/result.json b/fixtures/twin-desk-diff/contaminated/twin/stages/build/result.json new file mode 100644 index 0000000..be85c93 --- /dev/null +++ b/fixtures/twin-desk-diff/contaminated/twin/stages/build/result.json @@ -0,0 +1,5 @@ +{ + "stageId": "build", + "provider": "platformio", + "level": "not-run" +} diff --git a/fixtures/twin-desk-diff/contaminated/twin/stages/twin/result.json b/fixtures/twin-desk-diff/contaminated/twin/stages/twin/result.json new file mode 100644 index 0000000..3aae568 --- /dev/null +++ b/fixtures/twin-desk-diff/contaminated/twin/stages/twin/result.json @@ -0,0 +1,5 @@ +{ + "stageId": "twin", + "provider": "labwired-sim", + "level": "not-run" +} diff --git a/fixtures/twin-desk-diff/disagree/desk.receipt.json b/fixtures/twin-desk-diff/disagree/desk.receipt.json new file mode 100644 index 0000000..e73fc24 --- /dev/null +++ b/fixtures/twin-desk-diff/disagree/desk.receipt.json @@ -0,0 +1,7 @@ +{ + "bundle": "disagree/desk", + "side": "desk", + "artifactSha256": "1bd0221812559331be433e5b1e002945ebba25b6503b1fafc85adc4cd1555fcf", + "manifestSha256": "619741dc8f3ad9f2288c02195121a55a8b00511902b77844ebdbd20e17f95acc", + "result": "FAIL" +} diff --git a/fixtures/twin-desk-diff/disagree/desk/.owner.json b/fixtures/twin-desk-diff/disagree/desk/.owner.json new file mode 100644 index 0000000..be9044d --- /dev/null +++ b/fixtures/twin-desk-diff/disagree/desk/.owner.json @@ -0,0 +1,31 @@ +{ + "schema": 1, + "owner": "5adcaeb6-66ea-42cd-9dd8-b1426db70045", + "createdAt": "2026-08-16T16:11:24.476Z", + "targetIdentity": { + "id": "desk-c3", + "chip": "esp32c3" + }, + "behaviors": [ + { + "id": "heartbeat", + "provider": "serial", + "requiredLevel": "hardware_observed" + }, + { + "id": "led-blink", + "provider": "logic-csv", + "requiredLevel": "hardware_observed" + } + ], + "stages": [ + { + "id": "build", + "provider": "platformio" + }, + { + "id": "flash", + "provider": "probe-rs" + } + ] +} diff --git a/fixtures/twin-desk-diff/disagree/desk/observations/heartbeat.desk.capture.log b/fixtures/twin-desk-diff/disagree/desk/observations/heartbeat.desk.capture.log new file mode 100644 index 0000000..dff2af5 --- /dev/null +++ b/fixtures/twin-desk-diff/disagree/desk/observations/heartbeat.desk.capture.log @@ -0,0 +1,2 @@ +desk capture for heartbeat +level=hardware_observed diff --git a/fixtures/twin-desk-diff/disagree/desk/observations/heartbeat/result.json b/fixtures/twin-desk-diff/disagree/desk/observations/heartbeat/result.json new file mode 100644 index 0000000..adfdcaa --- /dev/null +++ b/fixtures/twin-desk-diff/disagree/desk/observations/heartbeat/result.json @@ -0,0 +1,28 @@ +{ + "behaviorId": "heartbeat", + "provider": "serial", + "level": "hardware_observed", + "artifactSha256": "1bd0221812559331be433e5b1e002945ebba25b6503b1fafc85adc4cd1555fcf", + "targetIdentity": { + "id": "desk-c3", + "chip": "esp32c3" + }, + "startedAt": "2026-08-16T16:11:24.288Z", + "endedAt": "2026-08-16T16:11:28.288Z", + "toolVersion": "probe-rs recorded fixture", + "rawEvidenceRefs": [ + "observations/heartbeat.desk.capture.log" + ], + "diagnostics": { + "detail": "desk observed heartbeat" + }, + "flashedArtifactSha256": "1bd0221812559331be433e5b1e002945ebba25b6503b1fafc85adc4cd1555fcf", + "rawEvidence": [ + { + "path": "observations/heartbeat.desk.capture.log", + "sha256": "5aba5981cb190ef4cd9d90f657261630d61d8845bcbc6885b24a4703c45f4bc6", + "size": 51 + } + ], + "requiredLevel": "hardware_observed" +} diff --git a/fixtures/twin-desk-diff/disagree/desk/observations/led-blink.desk.capture.log b/fixtures/twin-desk-diff/disagree/desk/observations/led-blink.desk.capture.log new file mode 100644 index 0000000..bb352c9 --- /dev/null +++ b/fixtures/twin-desk-diff/disagree/desk/observations/led-blink.desk.capture.log @@ -0,0 +1,3 @@ +desk capture for led-blink +level=failed +logic capture recorded 0 edges on the LED channel \ No newline at end of file diff --git a/fixtures/twin-desk-diff/disagree/desk/observations/led-blink/result.json b/fixtures/twin-desk-diff/disagree/desk/observations/led-blink/result.json new file mode 100644 index 0000000..a6e5a15 --- /dev/null +++ b/fixtures/twin-desk-diff/disagree/desk/observations/led-blink/result.json @@ -0,0 +1,19 @@ +{ + "behaviorId": "led-blink", + "provider": "logic-csv", + "level": "failed", + "rawEvidenceRefs": [ + "observations/led-blink.desk.capture.log" + ], + "diagnostics": { + "detail": "logic capture recorded 0 edges on the LED channel" + }, + "rawEvidence": [ + { + "path": "observations/led-blink.desk.capture.log", + "sha256": "17458f72c2e195e11fdad6d1e0a4420ed2229c383a97eb00192fca1c3be1c141", + "size": 89 + } + ], + "requiredLevel": "hardware_observed" +} diff --git a/fixtures/twin-desk-diff/disagree/desk/result.json b/fixtures/twin-desk-diff/disagree/desk/result.json new file mode 100644 index 0000000..d30b0ac --- /dev/null +++ b/fixtures/twin-desk-diff/disagree/desk/result.json @@ -0,0 +1,11 @@ +{ + "result": "FAIL", + "reasons": [ + { + "behaviorId": "led-blink", + "requiredLevel": "hardware_observed", + "actualLevel": "failed", + "reason": "required hardware_observed; recorded failed" + } + ] +} diff --git a/fixtures/twin-desk-diff/disagree/desk/stages/build/result.json b/fixtures/twin-desk-diff/disagree/desk/stages/build/result.json new file mode 100644 index 0000000..be85c93 --- /dev/null +++ b/fixtures/twin-desk-diff/disagree/desk/stages/build/result.json @@ -0,0 +1,5 @@ +{ + "stageId": "build", + "provider": "platformio", + "level": "not-run" +} diff --git a/fixtures/twin-desk-diff/disagree/desk/stages/flash/result.json b/fixtures/twin-desk-diff/disagree/desk/stages/flash/result.json new file mode 100644 index 0000000..b45d301 --- /dev/null +++ b/fixtures/twin-desk-diff/disagree/desk/stages/flash/result.json @@ -0,0 +1,5 @@ +{ + "stageId": "flash", + "provider": "probe-rs", + "level": "not-run" +} diff --git a/fixtures/twin-desk-diff/firmware-a.bin b/fixtures/twin-desk-diff/firmware-a.bin new file mode 100644 index 0000000..ac669f2 --- /dev/null +++ b/fixtures/twin-desk-diff/firmware-a.bin @@ -0,0 +1 @@ +labwired twin-desk differential firmware A diff --git a/fixtures/twin-desk-diff/firmware-b.bin b/fixtures/twin-desk-diff/firmware-b.bin new file mode 100644 index 0000000..98797e6 --- /dev/null +++ b/fixtures/twin-desk-diff/firmware-b.bin @@ -0,0 +1 @@ +labwired twin-desk differential firmware B diff --git a/fixtures/twin-desk-diff/mismatch/desk.receipt.json b/fixtures/twin-desk-diff/mismatch/desk.receipt.json new file mode 100644 index 0000000..ddb5a91 --- /dev/null +++ b/fixtures/twin-desk-diff/mismatch/desk.receipt.json @@ -0,0 +1,7 @@ +{ + "bundle": "mismatch/desk", + "side": "desk", + "artifactSha256": "52b17846bd3904e4b944589dc7dfbb70eda00f9d35da82e8747db9148bf77f78", + "manifestSha256": "a7091ae930795d60eebdb65caeea9fe0562a3c79b3894a1ebedb346200947a8d", + "result": "PASS" +} diff --git a/fixtures/twin-desk-diff/mismatch/desk/.owner.json b/fixtures/twin-desk-diff/mismatch/desk/.owner.json new file mode 100644 index 0000000..c7f392e --- /dev/null +++ b/fixtures/twin-desk-diff/mismatch/desk/.owner.json @@ -0,0 +1,31 @@ +{ + "schema": 1, + "owner": "29459c96-8cb2-458d-8dc6-437a5191ebbf", + "createdAt": "2026-08-16T16:11:24.645Z", + "targetIdentity": { + "id": "desk-c3", + "chip": "esp32c3" + }, + "behaviors": [ + { + "id": "heartbeat", + "provider": "serial", + "requiredLevel": "hardware_observed" + }, + { + "id": "led-blink", + "provider": "logic-csv", + "requiredLevel": "hardware_observed" + } + ], + "stages": [ + { + "id": "build", + "provider": "platformio" + }, + { + "id": "flash", + "provider": "probe-rs" + } + ] +} diff --git a/fixtures/twin-desk-diff/mismatch/desk/observations/heartbeat.desk.capture.log b/fixtures/twin-desk-diff/mismatch/desk/observations/heartbeat.desk.capture.log new file mode 100644 index 0000000..dff2af5 --- /dev/null +++ b/fixtures/twin-desk-diff/mismatch/desk/observations/heartbeat.desk.capture.log @@ -0,0 +1,2 @@ +desk capture for heartbeat +level=hardware_observed diff --git a/fixtures/twin-desk-diff/mismatch/desk/observations/heartbeat/result.json b/fixtures/twin-desk-diff/mismatch/desk/observations/heartbeat/result.json new file mode 100644 index 0000000..98f77a8 --- /dev/null +++ b/fixtures/twin-desk-diff/mismatch/desk/observations/heartbeat/result.json @@ -0,0 +1,28 @@ +{ + "behaviorId": "heartbeat", + "provider": "serial", + "level": "hardware_observed", + "artifactSha256": "52b17846bd3904e4b944589dc7dfbb70eda00f9d35da82e8747db9148bf77f78", + "targetIdentity": { + "id": "desk-c3", + "chip": "esp32c3" + }, + "startedAt": "2026-08-16T16:11:24.288Z", + "endedAt": "2026-08-16T16:11:28.288Z", + "toolVersion": "probe-rs recorded fixture", + "rawEvidenceRefs": [ + "observations/heartbeat.desk.capture.log" + ], + "diagnostics": { + "detail": "desk observed heartbeat" + }, + "flashedArtifactSha256": "52b17846bd3904e4b944589dc7dfbb70eda00f9d35da82e8747db9148bf77f78", + "rawEvidence": [ + { + "path": "observations/heartbeat.desk.capture.log", + "sha256": "5aba5981cb190ef4cd9d90f657261630d61d8845bcbc6885b24a4703c45f4bc6", + "size": 51 + } + ], + "requiredLevel": "hardware_observed" +} diff --git a/fixtures/twin-desk-diff/mismatch/desk/observations/led-blink.desk.capture.log b/fixtures/twin-desk-diff/mismatch/desk/observations/led-blink.desk.capture.log new file mode 100644 index 0000000..955cdb4 --- /dev/null +++ b/fixtures/twin-desk-diff/mismatch/desk/observations/led-blink.desk.capture.log @@ -0,0 +1,2 @@ +desk capture for led-blink +level=hardware_observed diff --git a/fixtures/twin-desk-diff/mismatch/desk/observations/led-blink/result.json b/fixtures/twin-desk-diff/mismatch/desk/observations/led-blink/result.json new file mode 100644 index 0000000..2326ca2 --- /dev/null +++ b/fixtures/twin-desk-diff/mismatch/desk/observations/led-blink/result.json @@ -0,0 +1,28 @@ +{ + "behaviorId": "led-blink", + "provider": "logic-csv", + "level": "hardware_observed", + "artifactSha256": "52b17846bd3904e4b944589dc7dfbb70eda00f9d35da82e8747db9148bf77f78", + "targetIdentity": { + "id": "desk-c3", + "chip": "esp32c3" + }, + "startedAt": "2026-08-16T16:11:24.288Z", + "endedAt": "2026-08-16T16:11:28.288Z", + "toolVersion": "probe-rs recorded fixture", + "rawEvidenceRefs": [ + "observations/led-blink.desk.capture.log" + ], + "diagnostics": { + "detail": "desk observed led-blink" + }, + "flashedArtifactSha256": "52b17846bd3904e4b944589dc7dfbb70eda00f9d35da82e8747db9148bf77f78", + "rawEvidence": [ + { + "path": "observations/led-blink.desk.capture.log", + "sha256": "5b3cacc1d608a94df1f8c95d3393e8e8c7c4f2a341da0048f2e1bdc1fbbe8fcf", + "size": 51 + } + ], + "requiredLevel": "hardware_observed" +} diff --git a/fixtures/twin-desk-diff/mismatch/desk/result.json b/fixtures/twin-desk-diff/mismatch/desk/result.json new file mode 100644 index 0000000..3426323 --- /dev/null +++ b/fixtures/twin-desk-diff/mismatch/desk/result.json @@ -0,0 +1,4 @@ +{ + "result": "PASS", + "reasons": [] +} diff --git a/fixtures/twin-desk-diff/mismatch/desk/stages/build/result.json b/fixtures/twin-desk-diff/mismatch/desk/stages/build/result.json new file mode 100644 index 0000000..be85c93 --- /dev/null +++ b/fixtures/twin-desk-diff/mismatch/desk/stages/build/result.json @@ -0,0 +1,5 @@ +{ + "stageId": "build", + "provider": "platformio", + "level": "not-run" +} diff --git a/fixtures/twin-desk-diff/mismatch/desk/stages/flash/result.json b/fixtures/twin-desk-diff/mismatch/desk/stages/flash/result.json new file mode 100644 index 0000000..b45d301 --- /dev/null +++ b/fixtures/twin-desk-diff/mismatch/desk/stages/flash/result.json @@ -0,0 +1,5 @@ +{ + "stageId": "flash", + "provider": "probe-rs", + "level": "not-run" +} diff --git a/fixtures/twin-desk-diff/no-probe/desk.receipt.json b/fixtures/twin-desk-diff/no-probe/desk.receipt.json new file mode 100644 index 0000000..c1b40ec --- /dev/null +++ b/fixtures/twin-desk-diff/no-probe/desk.receipt.json @@ -0,0 +1,7 @@ +{ + "bundle": "no-probe/desk", + "side": "desk", + "artifactSha256": "1bd0221812559331be433e5b1e002945ebba25b6503b1fafc85adc4cd1555fcf", + "manifestSha256": "e1d5f529d3589d53312bbce7769bcc1dae8e5acde2bb5c477d41b89efde1cac5", + "result": "FAIL" +} diff --git a/fixtures/twin-desk-diff/no-probe/desk/.owner.json b/fixtures/twin-desk-diff/no-probe/desk/.owner.json new file mode 100644 index 0000000..65e7e01 --- /dev/null +++ b/fixtures/twin-desk-diff/no-probe/desk/.owner.json @@ -0,0 +1,31 @@ +{ + "schema": 1, + "owner": "384803be-71c5-4e5e-87f3-2c1b50f509d4", + "createdAt": "2026-08-16T16:11:24.539Z", + "targetIdentity": { + "id": "desk-c3", + "chip": "esp32c3" + }, + "behaviors": [ + { + "id": "heartbeat", + "provider": "serial", + "requiredLevel": "hardware_observed" + }, + { + "id": "led-blink", + "provider": "logic-csv", + "requiredLevel": "hardware_observed" + } + ], + "stages": [ + { + "id": "build", + "provider": "platformio" + }, + { + "id": "flash", + "provider": "probe-rs" + } + ] +} diff --git a/fixtures/twin-desk-diff/no-probe/desk/observations/heartbeat.desk.capture.log b/fixtures/twin-desk-diff/no-probe/desk/observations/heartbeat.desk.capture.log new file mode 100644 index 0000000..efb7b0a --- /dev/null +++ b/fixtures/twin-desk-diff/no-probe/desk/observations/heartbeat.desk.capture.log @@ -0,0 +1,3 @@ +desk capture for heartbeat +level=blocked +no probe detected; exact flash was not proven \ No newline at end of file diff --git a/fixtures/twin-desk-diff/no-probe/desk/observations/heartbeat/result.json b/fixtures/twin-desk-diff/no-probe/desk/observations/heartbeat/result.json new file mode 100644 index 0000000..4732a42 --- /dev/null +++ b/fixtures/twin-desk-diff/no-probe/desk/observations/heartbeat/result.json @@ -0,0 +1,19 @@ +{ + "behaviorId": "heartbeat", + "provider": "serial", + "level": "blocked", + "rawEvidenceRefs": [ + "observations/heartbeat.desk.capture.log" + ], + "diagnostics": { + "detail": "no probe detected; exact flash was not proven" + }, + "rawEvidence": [ + { + "path": "observations/heartbeat.desk.capture.log", + "sha256": "592f6c590a7b160cf4d37496c7742bea4984867e87ab32ffdad4c425d7086b62", + "size": 86 + } + ], + "requiredLevel": "hardware_observed" +} diff --git a/fixtures/twin-desk-diff/no-probe/desk/observations/led-blink.desk.capture.log b/fixtures/twin-desk-diff/no-probe/desk/observations/led-blink.desk.capture.log new file mode 100644 index 0000000..a122776 --- /dev/null +++ b/fixtures/twin-desk-diff/no-probe/desk/observations/led-blink.desk.capture.log @@ -0,0 +1,3 @@ +desk capture for led-blink +level=blocked +no probe detected; exact flash was not proven \ No newline at end of file diff --git a/fixtures/twin-desk-diff/no-probe/desk/observations/led-blink/result.json b/fixtures/twin-desk-diff/no-probe/desk/observations/led-blink/result.json new file mode 100644 index 0000000..85d3fe5 --- /dev/null +++ b/fixtures/twin-desk-diff/no-probe/desk/observations/led-blink/result.json @@ -0,0 +1,19 @@ +{ + "behaviorId": "led-blink", + "provider": "logic-csv", + "level": "blocked", + "rawEvidenceRefs": [ + "observations/led-blink.desk.capture.log" + ], + "diagnostics": { + "detail": "no probe detected; exact flash was not proven" + }, + "rawEvidence": [ + { + "path": "observations/led-blink.desk.capture.log", + "sha256": "0a362b02e8b438c60db11855061c3580dc4e39ec5d98272d5c312c8ae3f8e572", + "size": 86 + } + ], + "requiredLevel": "hardware_observed" +} diff --git a/fixtures/twin-desk-diff/no-probe/desk/result.json b/fixtures/twin-desk-diff/no-probe/desk/result.json new file mode 100644 index 0000000..8afcc15 --- /dev/null +++ b/fixtures/twin-desk-diff/no-probe/desk/result.json @@ -0,0 +1,17 @@ +{ + "result": "FAIL", + "reasons": [ + { + "behaviorId": "heartbeat", + "requiredLevel": "hardware_observed", + "actualLevel": "blocked", + "reason": "required hardware_observed; recorded blocked" + }, + { + "behaviorId": "led-blink", + "requiredLevel": "hardware_observed", + "actualLevel": "blocked", + "reason": "required hardware_observed; recorded blocked" + } + ] +} diff --git a/fixtures/twin-desk-diff/no-probe/desk/stages/build/result.json b/fixtures/twin-desk-diff/no-probe/desk/stages/build/result.json new file mode 100644 index 0000000..be85c93 --- /dev/null +++ b/fixtures/twin-desk-diff/no-probe/desk/stages/build/result.json @@ -0,0 +1,5 @@ +{ + "stageId": "build", + "provider": "platformio", + "level": "not-run" +} diff --git a/fixtures/twin-desk-diff/no-probe/desk/stages/flash/result.json b/fixtures/twin-desk-diff/no-probe/desk/stages/flash/result.json new file mode 100644 index 0000000..b45d301 --- /dev/null +++ b/fixtures/twin-desk-diff/no-probe/desk/stages/flash/result.json @@ -0,0 +1,5 @@ +{ + "stageId": "flash", + "provider": "probe-rs", + "level": "not-run" +} diff --git a/lib/hardware/differential.mjs b/lib/hardware/differential.mjs new file mode 100644 index 0000000..ee2fc68 --- /dev/null +++ b/lib/hardware/differential.mjs @@ -0,0 +1,324 @@ +/** + * Differential twin-vs-desk comparison. + * + * One firmware artifact is taken to two independent targets: the LabWired + * digital twin and a real desk board. Each side produces its own authenticated + * evidence bundle. This module compares the two bundles and publishes the + * disagreement as a first-class result. + * + * Hard rules encoded here (see skills/desk-hw/SKILL.md): + * + * - A side's claim is computed from that side's bundle only. There is no code + * path that copies a level, a grade, or a pass from one side to the other, + * so hardware green can never be upgraded to twin green or the reverse. + * - `model_verified` is twin-only. It is minted from twin model evidence and + * never from flash, serial, or any desk record. + * - `hardware_observed` is desk-only. A desk record carrying a model level is + * inconclusive desk evidence, never a desk pass. + * - Both sides must bind to the exact same artifact digest. A mismatch is + * `invalid`; it is never smoothed into agreement. + * - Absent desk evidence is `desk-unavailable`, never a silent pass. + */ + +import fs from 'node:fs/promises'; +import path from 'node:path'; + +import { levelSatisfies, sha256File, verifyEvidenceBundle } from './evidence.mjs'; + +const SHA256 = /^[0-9a-f]{64}$/; + +/** Levels a twin bundle is permitted to record. */ +export const TWIN_LEVELS = Object.freeze(['compiled', 'model_observed', 'surrogate_model_observed']); +/** Levels a desk bundle is permitted to record as physical evidence. */ +export const DESK_LEVELS = Object.freeze(['hardware_observed']); +/** Levels that carry no verdict for either side. */ +export const INCONCLUSIVE_LEVELS = Object.freeze(['not-run', 'blocked', 'imported', 'untrusted_observation']); + +const TWIN_SET = new Set(TWIN_LEVELS); +const DESK_SET = new Set(DESK_LEVELS); +const INCONCLUSIVE_SET = new Set(INCONCLUSIVE_LEVELS); + +/** Process exit codes. Agreement is the only zero. */ +export const DIFFERENTIAL_EXIT_CODES = Object.freeze({ + agree: 0, + invalid: 2, + disagree: 3, + 'desk-unavailable': 4, + 'twin-unavailable': 5, +}); + +export function exitCodeForVerdict(verdict) { + const code = DIFFERENTIAL_EXIT_CODES[verdict]; + if (code === undefined) throw new TypeError(`unknown differential verdict ${verdict}`); + return code; +} + +function reason(code, message, extra = {}) { + return { code, message, ...extra }; +} + +async function readJsonFile(file) { + return JSON.parse(await fs.readFile(file, 'utf8')); +} + +/** + * Read the behavior records out of an evidence bundle that `verifyEvidenceBundle` + * has already re-derived byte for byte. Verification is the trust boundary; this + * is a plain read of the bytes it just authenticated. + */ +async function readAuthenticatedRecords(directory) { + const root = path.resolve(directory); + const owner = await readJsonFile(path.join(root, '.owner.json')); + const records = []; + for (const behavior of owner.behaviors) { + const record = await readJsonFile(path.join(root, 'observations', behavior.id, 'result.json')); + records.push(record); + } + return { owner, records }; +} + +/** + * Load one side of the comparison from a persisted evidence bundle. + * + * `side` is `twin` or `desk` and is never inferred from the bundle: the caller + * declares which target the bundle came from, and the bundle is then held to + * that side's level partition. + */ +export async function loadEvidenceSide(directory, { side, expectedManifestSha256 } = {}) { + if (side !== 'twin' && side !== 'desk') throw new TypeError('side must be twin or desk'); + if (typeof directory !== 'string' || directory === '') { + return { side, available: false, reasons: [reason('evidence_absent', `no ${side} evidence bundle was supplied`)] }; + } + const verification = await verifyEvidenceBundle(directory, { expectedManifestSha256 }); + if (!verification.valid) { + return { + side, + available: false, + authenticity: verification.authenticity ?? 'unverified', + reasons: [ + reason('evidence_unverified', `${side} evidence bundle did not authenticate`), + ...(verification.reasons ?? []), + ], + }; + } + const { owner, records } = await readAuthenticatedRecords(directory); + return { + side, + available: true, + authenticity: 'verified', + manifestSha256: verification.manifestSha256, + bundleResult: verification.result, + targetIdentity: owner.targetIdentity, + records, + reasons: [], + }; +} + +/** + * Classify one recorded behavior for one side. + * + * Only this side's record is consulted. `pass`/`fail` are behavioral verdicts; + * everything else is `inconclusive` and can never make the two sides agree. + */ +function classify(side, record) { + const level = record.level; + const allowed = side === 'twin' ? TWIN_SET : DESK_SET; + const base = { behaviorId: record.behaviorId, provider: record.provider, requiredLevel: record.requiredLevel, level }; + if (level === 'failed') { + return { ...base, outcome: 'fail', detail: 'recorded failed' }; + } + if (INCONCLUSIVE_SET.has(level)) { + return { ...base, outcome: 'inconclusive', detail: `recorded ${level}` }; + } + if (!allowed.has(level)) { + // A twin bundle claiming hardware evidence, or any unknown level. + return { ...base, outcome: 'contaminated', detail: `${side} side recorded ${level}, which only the other target may record` }; + } + if (side === 'twin' && level === 'compiled') { + return { ...base, outcome: 'inconclusive', detail: 'compiled_only is not behavior evidence' }; + } + if (!levelSatisfies(level, record.requiredLevel)) { + return { ...base, outcome: 'fail', detail: `required ${record.requiredLevel}; recorded ${level}` }; + } + return { ...base, outcome: 'pass', detail: `recorded ${level}` }; +} + +function artifactBindingReasons(side, record, artifactSha256) { + const problems = []; + if (record.artifactSha256 === undefined) return problems; + if (record.artifactSha256 !== artifactSha256) { + problems.push(reason('artifact_mismatch', + `${side} behavior ${record.behaviorId} is bound to a different artifact`, + { side, behaviorId: record.behaviorId, expected: artifactSha256, recorded: record.artifactSha256 })); + } + if (record.level === 'hardware_observed' && record.flashedArtifactSha256 !== artifactSha256) { + problems.push(reason('flashed_artifact_mismatch', + `${side} behavior ${record.behaviorId} did not flash the exact artifact`, + { side, behaviorId: record.behaviorId, expected: artifactSha256, recorded: record.flashedArtifactSha256 ?? null })); + } + return problems; +} + +/** + * Summarize one side without ever consulting the other side. + * `claim` is the publishable grade for this target alone. + */ +function summarizeSide(sideReport, artifactSha256) { + const side = sideReport.side; + if (!sideReport.available) { + return { + side, + available: false, + claim: null, + grade: null, + result: 'NOT_RUN', + behaviors: [], + reasons: sideReport.reasons ?? [], + }; + } + const behaviors = []; + const reasons = []; + for (const record of sideReport.records) { + reasons.push(...artifactBindingReasons(side, record, artifactSha256)); + behaviors.push(classify(side, record)); + } + const conclusive = behaviors.filter((item) => item.outcome === 'pass' || item.outcome === 'fail'); + const contaminated = behaviors.filter((item) => item.outcome === 'contaminated'); + for (const item of contaminated) { + reasons.push(reason('level_partition_violated', item.detail, { side, behaviorId: item.behaviorId, level: item.level })); + } + const allPass = conclusive.length > 0 && conclusive.every((item) => item.outcome === 'pass'); + let grade = null; + if (allPass && contaminated.length === 0) { + const levels = new Set(conclusive.map((item) => item.level)); + grade = side === 'twin' + ? (levels.has('surrogate_model_observed') ? 'surrogate_model_observed' : 'model_observed') + : 'hardware_observed'; + } + // `model_verified` is minted only from twin model evidence; `hardware_observed` + // only from desk physical evidence. Neither branch can read the other side. + const claim = grade === null ? null : (side === 'twin' ? 'model_verified' : 'hardware_observed'); + return { + side, + available: true, + authenticity: sideReport.authenticity, + manifestSha256: sideReport.manifestSha256, + targetIdentity: sideReport.targetIdentity, + claim, + grade, + result: conclusive.length === 0 ? 'INCONCLUSIVE' : (allPass ? 'PASS' : 'FAIL'), + behaviors, + reasons, + }; +} + +function byId(behaviors) { + return new Map(behaviors.map((item) => [item.behaviorId, item])); +} + +/** + * Compare an authenticated twin side against an authenticated desk side. + * + * Pure: it reads only the two summaries and the artifact digest, so the same + * inputs are reproducible from recorded fixtures and from live runs alike. + */ +export function diffTwinDesk({ artifactSha256, artifactPath = null, twin, desk }) { + if (typeof artifactSha256 !== 'string' || !SHA256.test(artifactSha256)) { + throw new TypeError('artifactSha256 must be a lowercase SHA-256 digest'); + } + const twinSummary = summarizeSide(twin, artifactSha256); + const deskSummary = summarizeSide(desk, artifactSha256); + + const invalid = [ + ...twinSummary.reasons.filter((item) => item.code === 'artifact_mismatch' || item.code === 'flashed_artifact_mismatch' || item.code === 'level_partition_violated'), + ...deskSummary.reasons.filter((item) => item.code === 'artifact_mismatch' || item.code === 'flashed_artifact_mismatch' || item.code === 'level_partition_violated'), + ]; + + const twinBehaviors = byId(twinSummary.behaviors); + const deskBehaviors = byId(deskSummary.behaviors); + const paired = []; + const unpaired = []; + for (const [behaviorId, twinItem] of twinBehaviors) { + const deskItem = deskBehaviors.get(behaviorId); + if (!deskItem) { + unpaired.push({ behaviorId, presentOn: 'twin', twin: twinItem, desk: null }); + continue; + } + const comparable = (twinItem.outcome === 'pass' || twinItem.outcome === 'fail') + && (deskItem.outcome === 'pass' || deskItem.outcome === 'fail'); + paired.push({ + behaviorId, + twin: twinItem, + desk: deskItem, + agreement: comparable ? (twinItem.outcome === deskItem.outcome ? 'agree' : 'disagree') : 'incomparable', + }); + } + for (const [behaviorId, deskItem] of deskBehaviors) { + if (!twinBehaviors.has(behaviorId)) unpaired.push({ behaviorId, presentOn: 'desk', twin: null, desk: deskItem }); + } + const disagreements = paired.filter((item) => item.agreement === 'disagree'); + const agreements = paired.filter((item) => item.agreement === 'agree'); + const comparablePairs = agreements.length + disagreements.length; + + const reasons = []; + let verdict; + if (invalid.length > 0) { + verdict = 'invalid'; + reasons.push(...invalid); + } else if (!twinSummary.available || twinSummary.result === 'INCONCLUSIVE') { + verdict = 'twin-unavailable'; + reasons.push(...(twinSummary.reasons.length ? twinSummary.reasons + : [reason('twin_inconclusive', 'the twin bundle recorded no conclusive behavior')])); + } else if (!deskSummary.available) { + verdict = 'desk-unavailable'; + reasons.push(...(deskSummary.reasons.length ? deskSummary.reasons + : [reason('evidence_absent', 'no desk evidence bundle was supplied')])); + } else if (deskSummary.result === 'INCONCLUSIVE') { + verdict = 'desk-unavailable'; + reasons.push(reason('desk_inconclusive', + 'the desk bundle recorded no conclusive physical behavior; no board evidence exists for this artifact')); + } else if (comparablePairs === 0) { + verdict = 'invalid'; + reasons.push(reason('no_comparable_behavior', + 'the twin and desk bundles share no behavior that both sides decided')); + } else if (disagreements.length > 0) { + verdict = 'disagree'; + for (const item of disagreements) { + reasons.push(reason('behavior_disagreement', + `${item.behaviorId}: twin ${item.twin.outcome} (${item.twin.level}), desk ${item.desk.outcome} (${item.desk.level})`, + { behaviorId: item.behaviorId, twinOutcome: item.twin.outcome, deskOutcome: item.desk.outcome })); + } + } else { + verdict = 'agree'; + } + + return { + schema: 1, + kind: 'twin-desk-differential', + verdict, + artifact: { sha256: artifactSha256, path: artifactPath }, + twin: twinSummary, + desk: deskSummary, + comparison: { paired, unpaired, comparablePairs, agreed: agreements.length, disagreed: disagreements.length }, + disagreements: disagreements.map((item) => ({ + behaviorId: item.behaviorId, + twin: { outcome: item.twin.outcome, level: item.twin.level, detail: item.twin.detail }, + desk: { outcome: item.desk.outcome, level: item.desk.level, detail: item.desk.detail }, + })), + reasons, + exitCode: exitCodeForVerdict(verdict), + }; +} + +/** + * End-to-end differential over one firmware artifact and two recorded or live + * evidence bundles. Omitting the desk bundle (no probe, no board) degrades to + * `desk-unavailable`; it never fabricates a comparison. + */ +export async function runDifferential({ artifactPath, twin = {}, desk = {} } = {}) { + if (typeof artifactPath !== 'string' || artifactPath === '') throw new TypeError('artifactPath is required'); + const artifactSha256 = (await sha256File(artifactPath)).toLowerCase(); + const twinSide = await loadEvidenceSide(twin.evidenceDir, { side: 'twin', expectedManifestSha256: twin.receipt }); + const deskSide = await loadEvidenceSide(desk.evidenceDir, { side: 'desk', expectedManifestSha256: desk.receipt }); + return diffTwinDesk({ artifactSha256, artifactPath: path.resolve(artifactPath), twin: twinSide, desk: deskSide }); +} diff --git a/package.json b/package.json index bef30c0..66a11de 100644 --- a/package.json +++ b/package.json @@ -159,7 +159,7 @@ "test:develop:agent": "bash tests/develop-agent-e2e.sh", "test:develop:release": "LABWIRED_ACCEPTANCE_REQUIRE_COMPLETE=1 bash tests/develop-agent-e2e.sh", "test:dispatcher": "bash tests/dispatcher.sh", - "test:node18-min": "node tests/hardware-cli-node.test.mjs && node tests/hardware-runner.test.mjs", + "test:node18-min": "node tests/hardware-cli-node.test.mjs && node tests/hardware-runner.test.mjs && node tests/hardware-differential.test.mjs", "test:agent-lifecycle": "bash tests/agent-lifecycle.sh", "test:tool-names": "bash tests/public-tool-names.sh", "test:public-install-safety": "bash tests/public-install-safety.sh", diff --git a/scripts/hardware-runner.mjs b/scripts/hardware-runner.mjs index e49c436..3f91297 100644 --- a/scripts/hardware-runner.mjs +++ b/scripts/hardware-runner.mjs @@ -1,9 +1,11 @@ #!/usr/bin/env node import { createHash } from 'node:crypto'; import { statSync } from 'node:fs'; +import { writeFile } from 'node:fs/promises'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; +import { runDifferential } from '../lib/hardware/differential.mjs'; import { executeHardwareRun, planHardwareRun } from '../lib/hardware/runner.mjs'; import { resolveLaunch, runLaunch } from '../lib/hardware/process.mjs'; import { containsInlineCredential } from '../lib/hardware/profile.mjs'; @@ -17,9 +19,44 @@ function fail(message, usage = false) { throw error; } +const DIFF_USAGE = 'usage: hardware diff --artifact FILE --twin-evidence DIR --twin-receipt SHA256 ' + + '[--desk-evidence DIR --desk-receipt SHA256] [--out FILE]'; + +export function parseDifferentialArguments(rest) { + const allowed = new Set(['--artifact', '--twin-evidence', '--twin-receipt', '--desk-evidence', '--desk-receipt', '--out']); + const values = {}; + for (let index = 0; index < rest.length; index += 2) { + const flag = rest[index]; + if (!allowed.has(flag) || index + 1 >= rest.length || rest[index + 1].startsWith('--')) fail(`unsupported or incomplete hardware option ${flag ?? ''}`, true); + if (values[flag] !== undefined) fail(`duplicate hardware option ${flag}`, true); + values[flag] = rest[index + 1]; + } + if (!values['--artifact']) fail(`--artifact is required; ${DIFF_USAGE}`, true); + if (Object.values(values).some((value) => containsInlineCredential(value))) { + fail('hardware arguments must not contain credential-shaped values', true); + } + for (const flag of ['--twin-receipt', '--desk-receipt']) { + if (values[flag] !== undefined && !SHA256.test(values[flag])) { + fail(`${flag} must be the exact lowercase evidence receipt digest`, true); + } + } + // A bundle without its out-of-bundle receipt cannot be authenticated, so it is + // never silently accepted as evidence for its side. + if (values['--twin-evidence'] && !values['--twin-receipt']) fail('--twin-evidence requires --twin-receipt', true); + if (values['--desk-evidence'] && !values['--desk-receipt']) fail('--desk-evidence requires --desk-receipt', true); + return Object.freeze({ + command: 'diff', + artifactPath: values['--artifact'], + twin: Object.freeze({ evidenceDir: values['--twin-evidence'], receipt: values['--twin-receipt'] }), + desk: Object.freeze({ evidenceDir: values['--desk-evidence'], receipt: values['--desk-receipt'] }), + outPath: values['--out'], + }); +} + export function parseHardwareArguments(argv) { const [command, ...rest] = argv; - if (!['plan', 'run'].includes(command)) fail('usage: hardware plan|run --profile FILE --out DIR [--confirm DIGEST]', true); + if (command === 'diff') return parseDifferentialArguments(rest); + if (!['plan', 'run'].includes(command)) fail('usage: hardware plan|run --profile FILE --out DIR [--confirm DIGEST]\n hardware diff --artifact FILE --twin-evidence DIR --twin-receipt SHA256 [--desk-evidence DIR --desk-receipt SHA256]', true); const allowed = new Set(command === 'plan' ? ['--profile', '--out'] : ['--profile', '--out', '--confirm']); const values = {}; for (let index = 0; index < rest.length; index += 2) { @@ -205,6 +242,9 @@ export async function resolveHardwareIdentities(profile, options = {}) { export async function main(argv = process.argv.slice(2)) { let parsed; + // The requested verb is known before parsing, so a diff invocation that fails + // to parse still reports as a diff and still avoids the "disagree" exit code. + const requestedDiff = argv[0] === 'diff'; try { if (Number(process.versions.node.split('.')[0]) < 18) fail('Node.js 18+ is required for hardware commands', true); parsed = parseHardwareArguments(argv); @@ -213,6 +253,13 @@ export async function main(argv = process.argv.slice(2)) { process.once('SIGINT', stop); process.once('SIGTERM', stop); const dependencies = { resolveHardwareIdentities }; try { + if (parsed.command === 'diff') { + const diff = await runDifferential(parsed); + const payload = `${JSON.stringify({ command: 'hardware diff', ...diff })}\n`; + process.stdout.write(payload); + if (parsed.outPath) await writeFile(parsed.outPath, payload); + return diff.exitCode; + } if (parsed.command === 'plan') { const result = await planHardwareRun({ ...parsed, dependencies, signal: abort.signal }); process.stdout.write(`${JSON.stringify({ command: 'hardware plan', ...result })}\n`); @@ -228,7 +275,15 @@ export async function main(argv = process.argv.slice(2)) { const message = String(error?.message ?? error); const confirmation = /confirmation digest/i.test(message); const blocked = confirmation || /^BLOCKED:|exactly one unique detected identity|ambiguous .* identity/i.test(message); - process.stdout.write(`${JSON.stringify({ command: `hardware ${parsed?.command ?? 'invalid'}`, result: blocked ? 'BLOCKED' : 'FAIL', error: message })}\n`); + process.stdout.write(`${JSON.stringify({ + command: `hardware ${parsed?.command ?? 'invalid'}`, + result: blocked ? 'BLOCKED' : 'FAIL', + ...(requestedDiff ? { command: 'hardware diff', verdict: 'invalid' } : {}), + error: message, + })}\n`); + // Exit 3 means "disagree" for diff, so a broken diff invocation must never + // borrow it. Every diff failure is invalid (2). + if (requestedDiff) return 2; return parsed?.command === 'run' && confirmation ? 2 : (error?.usage ? 2 : 3); } } diff --git a/skills/desk-hw/SKILL.md b/skills/desk-hw/SKILL.md index 59812de..c3c0129 100644 --- a/skills/desk-hw/SKILL.md +++ b/skills/desk-hw/SKILL.md @@ -93,6 +93,22 @@ hardware_status: evidence_receipt: ``` +## E. Differential: same firmware, twin and desk + +When both lanes ran the same artifact, compare their authenticated bundles: + +```bash +labwired agent hardware diff --artifact \ + --twin-evidence --twin-receipt \ + --desk-evidence --desk-receipt +``` + +Exit `0` agree, `3` disagree, `4` desk-unavailable, `5` twin-unavailable, +`2` invalid. A disagreement is a first-class result to publish, never an error +to smooth over. Each side keeps its own grade: the diff never upgrades a +hardware green to a twin green or the reverse, and a missing probe yields +`desk-unavailable`, never a pass. + ## USB-CDC notes ESP32/RP2040 may re-enumerate after reset; re-resolve port. Baud must match firmware. diff --git a/tests/hardware-differential.test.mjs b/tests/hardware-differential.test.mjs new file mode 100644 index 0000000..4ccdd6f --- /dev/null +++ b/tests/hardware-differential.test.mjs @@ -0,0 +1,242 @@ +import assert from 'node:assert/strict'; +import { spawn } from 'node:child_process'; +import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import test from 'node:test'; + +import { diffTwinDesk, exitCodeForVerdict, loadEvidenceSide, runDifferential } from '../lib/hardware/differential.mjs'; + +const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); +const FIXTURES = path.join(ROOT, 'fixtures/twin-desk-diff'); +const RUNNER = path.join(ROOT, 'scripts/hardware-runner.mjs'); +const ARTIFACT_A = path.join(FIXTURES, 'firmware-a.bin'); +const ARTIFACT_B = path.join(FIXTURES, 'firmware-b.bin'); + +async function receipt(name) { + const data = JSON.parse(await readFile(path.join(FIXTURES, `${name}.receipt.json`), 'utf8')); + return data.manifestSha256; +} + +function bundle(name) { + return path.join(FIXTURES, name); +} + +async function diff(twinName, deskName, { artifact = ARTIFACT_A } = {}) { + return runDifferential({ + artifactPath: artifact, + twin: twinName ? { evidenceDir: bundle(twinName), receipt: await receipt(twinName) } : {}, + desk: deskName ? { evidenceDir: bundle(deskName), receipt: await receipt(deskName) } : {}, + }); +} + +function runCli(args) { + return new Promise((resolve, reject) => { + const child = spawn(process.execPath, [RUNNER, ...args], { cwd: ROOT, stdio: ['ignore', 'pipe', 'pipe'] }); + let stdout = ''; let stderr = ''; + child.stdout.on('data', (chunk) => { stdout += chunk; }); + child.stderr.on('data', (chunk) => { stderr += chunk; }); + child.on('error', reject); + child.on('close', (code) => resolve({ code, stdout, stderr })); + }); +} + +test('recorded twin and desk bundles that both observed the firmware agree', async () => { + const result = await diff('agree/twin', 'agree/desk'); + assert.equal(result.verdict, 'agree'); + assert.equal(result.exitCode, 0); + assert.equal(result.comparison.comparablePairs, 2); + assert.equal(result.comparison.disagreed, 0); + assert.equal(result.disagreements.length, 0); +}); + +test('each side keeps its own evidence grade and neither is upgraded from the other', async () => { + const result = await diff('agree/twin', 'agree/desk'); + assert.equal(result.twin.claim, 'model_verified'); + assert.equal(result.twin.grade, 'model_observed'); + assert.equal(result.desk.claim, 'hardware_observed'); + assert.equal(result.desk.grade, 'hardware_observed'); + assert.equal(result.artifact.sha256, result.twin.behaviors.length ? result.artifact.sha256 : null); + for (const item of result.twin.behaviors) assert.notEqual(item.level, 'hardware_observed'); + for (const item of result.desk.behaviors) assert.equal(item.level, 'hardware_observed'); +}); + +test('a twin green and a desk red on the same behavior is published as a disagreement', async () => { + const result = await diff('agree/twin', 'disagree/desk'); + assert.equal(result.verdict, 'disagree'); + assert.equal(result.exitCode, 3); + assert.deepEqual(result.disagreements.map((item) => item.behaviorId), ['led-blink']); + const led = result.disagreements[0]; + assert.equal(led.twin.outcome, 'pass'); + assert.equal(led.desk.outcome, 'fail'); + // The twin still publishes its own honest claim; the desk still publishes its own. + assert.equal(result.twin.claim, 'model_verified'); + assert.equal(result.desk.claim, null); + assert.equal(result.desk.result, 'FAIL'); +}); + +test('no desk bundle at all degrades to desk-unavailable, never to a pass', async () => { + const result = await diff('agree/twin', null); + assert.equal(result.verdict, 'desk-unavailable'); + assert.equal(result.exitCode, 4); + assert.notEqual(result.exitCode, 0); + assert.equal(result.desk.available, false); + assert.equal(result.desk.claim, null); + assert.ok(result.reasons.some((item) => item.code === 'evidence_absent')); +}); + +test('a desk bundle with no probe records no physical evidence and stays desk-unavailable', async () => { + const result = await diff('agree/twin', 'no-probe/desk'); + assert.equal(result.verdict, 'desk-unavailable'); + assert.equal(result.exitCode, 4); + assert.equal(result.desk.result, 'INCONCLUSIVE'); + assert.equal(result.desk.claim, null); + assert.ok(result.reasons.some((item) => item.code === 'desk_inconclusive')); +}); + +test('a twin bundle that claims hardware evidence is rejected, not accepted as desk green', async () => { + const result = await diff('contaminated/twin', 'agree/desk'); + assert.equal(result.verdict, 'invalid'); + assert.equal(result.exitCode, 2); + assert.ok(result.reasons.some((item) => item.code === 'level_partition_violated')); + assert.equal(result.twin.claim, null); +}); + +test('a desk bundle bound to different firmware is invalid, never agreement', async () => { + const result = await diff('agree/twin', 'mismatch/desk'); + assert.equal(result.verdict, 'invalid'); + assert.equal(result.exitCode, 2); + assert.ok(result.reasons.some((item) => item.code === 'artifact_mismatch')); +}); + +test('a bundle without its out-of-bundle receipt does not authenticate', async () => { + const side = await loadEvidenceSide(bundle('agree/desk'), { side: 'desk' }); + assert.equal(side.available, false); + assert.ok(side.reasons.some((item) => item.code === 'receipt_required' || item.code === 'evidence_unverified')); + const result = await runDifferential({ + artifactPath: ARTIFACT_A, + twin: { evidenceDir: bundle('agree/twin'), receipt: await receipt('agree/twin') }, + desk: { evidenceDir: bundle('agree/desk') }, + }); + assert.equal(result.verdict, 'desk-unavailable'); + assert.equal(result.exitCode, 4); +}); + +test('a tampered receipt does not authenticate the bundle', async () => { + const wrong = 'f'.repeat(64); + const result = await runDifferential({ + artifactPath: ARTIFACT_A, + twin: { evidenceDir: bundle('agree/twin'), receipt: await receipt('agree/twin') }, + desk: { evidenceDir: bundle('agree/desk'), receipt: wrong }, + }); + assert.equal(result.verdict, 'desk-unavailable'); + assert.ok(result.desk.reasons.some((item) => item.code === 'receipt_mismatch')); +}); + +test('the whole diff refuses the wrong artifact instead of comparing anyway', async () => { + const result = await diff('agree/twin', 'agree/desk', { artifact: ARTIFACT_B }); + assert.equal(result.verdict, 'invalid'); + assert.ok(result.reasons.every((item) => item.code !== 'behavior_disagreement')); +}); + +test('agreement requires at least one behavior both sides decided', () => { + const empty = { side: 'twin', available: true, authenticity: 'verified', records: [] }; + const desk = { side: 'desk', available: true, authenticity: 'verified', records: [] }; + const result = diffTwinDesk({ artifactSha256: 'a'.repeat(64), twin: empty, desk }); + assert.notEqual(result.verdict, 'agree'); + assert.equal(result.verdict, 'twin-unavailable'); + assert.equal(result.exitCode, 5); +}); + +test('exit codes are distinct and only agreement is zero', () => { + const codes = ['agree', 'disagree', 'desk-unavailable', 'twin-unavailable', 'invalid'].map(exitCodeForVerdict); + assert.equal(new Set(codes).size, codes.length); + assert.equal(exitCodeForVerdict('agree'), 0); + for (const verdict of ['disagree', 'desk-unavailable', 'twin-unavailable', 'invalid']) { + assert.notEqual(exitCodeForVerdict(verdict), 0); + } +}); + +test('CLI exits 0 on agreement and writes the structured diff', async () => { + const scratch = await mkdtemp(path.join(os.tmpdir(), 'labwired-diff-')); + try { + const out = path.join(scratch, 'diff.json'); + const result = await runCli([ + 'diff', '--artifact', ARTIFACT_A, + '--twin-evidence', bundle('agree/twin'), '--twin-receipt', await receipt('agree/twin'), + '--desk-evidence', bundle('agree/desk'), '--desk-receipt', await receipt('agree/desk'), + '--out', out, + ]); + assert.equal(result.code, 0); + const written = JSON.parse(await readFile(out, 'utf8')); + assert.equal(written.command, 'hardware diff'); + assert.equal(written.verdict, 'agree'); + assert.equal(written.artifact.sha256.length, 64); + } finally { + await rm(scratch, { recursive: true, force: true }); + } +}); + +test('CLI exits 3 on a published disagreement', async () => { + const result = await runCli([ + 'diff', '--artifact', ARTIFACT_A, + '--twin-evidence', bundle('agree/twin'), '--twin-receipt', await receipt('agree/twin'), + '--desk-evidence', bundle('disagree/desk'), '--desk-receipt', await receipt('disagree/desk'), + ]); + assert.equal(result.code, 3); + assert.equal(JSON.parse(result.stdout).verdict, 'disagree'); +}); + +test('CLI exits 4 when no board is attached', async () => { + const result = await runCli([ + 'diff', '--artifact', ARTIFACT_A, + '--twin-evidence', bundle('agree/twin'), '--twin-receipt', await receipt('agree/twin'), + ]); + assert.equal(result.code, 4); + assert.equal(JSON.parse(result.stdout).verdict, 'desk-unavailable'); +}); + +test('CLI refuses a desk bundle supplied without its receipt', async () => { + const result = await runCli([ + 'diff', '--artifact', ARTIFACT_A, + '--twin-evidence', bundle('agree/twin'), '--twin-receipt', await receipt('agree/twin'), + '--desk-evidence', bundle('agree/desk'), + ]); + assert.equal(result.code, 2); + assert.equal(JSON.parse(result.stdout).verdict, 'invalid'); +}); + +test('CLI exits 2, never 3, when the artifact is missing', async () => { + const scratch = await mkdtemp(path.join(os.tmpdir(), 'labwired-diff-')); + try { + const result = await runCli([ + 'diff', '--artifact', path.join(scratch, 'absent.bin'), + '--twin-evidence', bundle('agree/twin'), '--twin-receipt', await receipt('agree/twin'), + ]); + assert.equal(result.code, 2); + } finally { + await rm(scratch, { recursive: true, force: true }); + } +}); + +test('an edited desk record breaks bundle authentication instead of changing the verdict', async () => { + const scratch = await mkdtemp(path.join(os.tmpdir(), 'labwired-diff-')); + try { + const copy = path.join(scratch, 'desk'); + await import('node:fs/promises').then(({ cp }) => cp(bundle('disagree/desk'), copy, { recursive: true })); + const recordPath = path.join(copy, 'observations/led-blink/result.json'); + const record = JSON.parse(await readFile(recordPath, 'utf8')); + record.diagnostics = { detail: 'hand edited' }; + await writeFile(recordPath, `${JSON.stringify(record, null, 2)}\n`); + const result = await runDifferential({ + artifactPath: ARTIFACT_A, + twin: { evidenceDir: bundle('agree/twin'), receipt: await receipt('agree/twin') }, + desk: { evidenceDir: copy, receipt: await receipt('disagree/desk') }, + }); + assert.equal(result.verdict, 'desk-unavailable'); + assert.equal(result.desk.available, false); + } finally { + await rm(scratch, { recursive: true, force: true }); + } +}); diff --git a/tests/hardware-public-docs.sh b/tests/hardware-public-docs.sh index 3713a31..ffd2f36 100644 --- a/tests/hardware-public-docs.sh +++ b/tests/hardware-public-docs.sh @@ -42,6 +42,16 @@ require_text docs/USAGE.md 'Ordinary.*BLOCKED' 'ordinary blocked exit behavior i require_text docs/USAGE.md 'results.*exit `3`' 'ordinary blocked/fail exit code is undocumented' require_text scripts/hardware-runner.mjs "return parsed\?\.command === 'run'.*\? 2.*: 3" 'documented exit meanings are not tied to runner constants' +require_text docs/USAGE.md 'hardware diff --artifact' 'twin-vs-desk differential command is undocumented' +for verdict in agree disagree desk-unavailable twin-unavailable invalid; do + require_text docs/USAGE.md "$verdict" "differential verdict ${verdict} is undocumented" + require_text lib/hardware/differential.mjs "$verdict" "differential verdict ${verdict} is not a runner constant" +done +require_text docs/USAGE.md 'never.*silent pass|silent pass' 'differential absent-desk honesty is undocumented' +require_text scripts/hardware-runner.mjs 'if \(requestedDiff\) return 2;' 'a failed diff can still borrow the disagree exit code' +require_text skills/desk-hw/SKILL.md 'hardware diff' 'desk-hw omits the twin-vs-desk differential' +require_text skills/desk-hw/SKILL.md 'disagreement' 'desk-hw does not publish twin-vs-desk disagreement' + require_text docs/TESTING.md 'hardware-release-contract' 'release hardware lane is undocumented' require_text docs/TESTING.md 'windows-hardware-contract' 'Windows hardware lane is undocumented' require_text docs/TESTING.md 'probe-exact-flash' 'exact flash lane is undocumented' diff --git a/tests/helpers/make-twin-desk-fixtures.mjs b/tests/helpers/make-twin-desk-fixtures.mjs new file mode 100644 index 0000000..f6a1f64 --- /dev/null +++ b/tests/helpers/make-twin-desk-fixtures.mjs @@ -0,0 +1,143 @@ +#!/usr/bin/env node +/** + * Record the twin-vs-desk differential fixtures once. + * + * These are genuine evidence bundles produced by `createEvidenceBundle`, not + * hand-written JSON, so the committed fixtures exercise the same authentication + * path a live run does. Regenerate only when the evidence format changes: + * + * node tests/helpers/make-twin-desk-fixtures.mjs + * + * Bundle timestamps are fixed in the past. `verifyEvidenceBundle` only bounds + * timestamps from above, so recorded bundles stay verifiable indefinitely. + */ + +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +import { createEvidenceBundle, sha256File } from '../../lib/hardware/evidence.mjs'; + +const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../..'); +const OUT = path.join(ROOT, 'fixtures/twin-desk-diff'); + +const TARGET = Object.freeze({ id: 'desk-c3', chip: 'esp32c3' }); +const BEHAVIORS = Object.freeze(['heartbeat', 'led-blink']); +// Recorded at generation time so each bundle's claims sit inside its own +// `.owner.json` run window. Verification only bounds timestamps from above, so +// the committed fixtures stay verifiable indefinitely. +const RECORDED_AT = Date.now(); +const STARTED_AT = new Date(RECORDED_AT).toISOString(); +const ENDED_AT = new Date(RECORDED_AT + 4_000).toISOString(); + +function observations(requiredLevel) { + return [ + { id: 'heartbeat', provider: 'serial', requiredLevel }, + { id: 'led-blink', provider: 'logic-csv', requiredLevel }, + ]; +} + +async function writeRaw(bundle, name, body) { + const file = path.join(bundle.root, 'observations', name); + await fs.writeFile(file, body); + return `observations/${name}`; +} + +/** + * @param {string} name bundle directory under fixtures/twin-desk-diff + * @param {'twin'|'desk'} side + * @param {string} artifactSha256 + * @param {Record} outcomes keyed by behavior id + */ +async function record(name, side, artifactSha256, outcomes) { + const directory = path.join(OUT, name); + await fs.rm(directory, { recursive: true, force: true }); + const requiredLevel = side === 'twin' ? 'model_observed' : 'hardware_observed'; + const profile = { target: TARGET, observations: observations(requiredLevel) }; + const bundle = await createEvidenceBundle(directory, profile, { + stages: side === 'twin' + ? [{ id: 'build', provider: 'platformio' }, { id: 'twin', provider: 'labwired-sim' }] + : [{ id: 'build', provider: 'platformio' }, { id: 'flash', provider: 'probe-rs' }], + }); + for (const behaviorId of BEHAVIORS) { + const outcome = outcomes[behaviorId]; + const reference = await writeRaw(bundle, `${behaviorId}.${side}.capture.log`, + `${side} capture for ${behaviorId}\nlevel=${outcome.level}\n${outcome.raw ?? ''}`); + if (outcome.level === 'failed' || outcome.level === 'blocked') { + await bundle.recordBehavior(behaviorId, { + behaviorId, + provider: profile.observations.find((item) => item.id === behaviorId).provider, + level: outcome.level, + rawEvidenceRefs: [reference], + diagnostics: { detail: outcome.raw ?? `${side} did not establish ${behaviorId}` }, + }); + continue; + } + const common = { + behaviorId, + provider: profile.observations.find((item) => item.id === behaviorId).provider, + level: outcome.level, + artifactSha256, + targetIdentity: { ...TARGET }, + startedAt: STARTED_AT, + endedAt: ENDED_AT, + toolVersion: side === 'twin' ? 'labwired-sim recorded fixture' : 'probe-rs recorded fixture', + rawEvidenceRefs: [reference], + diagnostics: { detail: outcome.raw ?? `${side} observed ${behaviorId}` }, + }; + if (outcome.level === 'model_observed') common.nativeArtifactSha256 = artifactSha256; + if (outcome.level === 'hardware_observed') common.flashedArtifactSha256 = artifactSha256; + await bundle.recordBehavior(behaviorId, common); + } + const receipt = await bundle.finalize(); + await fs.writeFile(path.join(OUT, `${name}.receipt.json`), + `${JSON.stringify({ bundle: name, side, artifactSha256, manifestSha256: receipt.manifestSha256, result: receipt.result }, null, 2)}\n`); + process.stdout.write(`recorded ${name} (${side}) result=${receipt.result} manifest=${receipt.manifestSha256}\n`); + return receipt; +} + +async function main() { + await fs.rm(OUT, { recursive: true, force: true }); + await fs.mkdir(OUT, { recursive: true }); + const artifactA = path.join(OUT, 'firmware-a.bin'); + const artifactB = path.join(OUT, 'firmware-b.bin'); + await fs.writeFile(artifactA, 'labwired twin-desk differential firmware A\n'); + await fs.writeFile(artifactB, 'labwired twin-desk differential firmware B\n'); + const shaA = await sha256File(artifactA); + const shaB = await sha256File(artifactB); + + // Both targets ran firmware A and both saw both behaviors. + await record('agree/twin', 'twin', shaA, { + heartbeat: { level: 'model_observed' }, 'led-blink': { level: 'model_observed' }, + }); + await record('agree/desk', 'desk', shaA, { + heartbeat: { level: 'hardware_observed' }, 'led-blink': { level: 'hardware_observed' }, + }); + + // The twin says the LED blinks; the desk board says it does not. This is the + // published disagreement. + await record('disagree/desk', 'desk', shaA, { + heartbeat: { level: 'hardware_observed' }, + 'led-blink': { level: 'failed', raw: 'logic capture recorded 0 edges on the LED channel' }, + }); + + // No probe was detected, so nothing physical was observed at all. + await record('no-probe/desk', 'desk', shaA, { + heartbeat: { level: 'blocked', raw: 'no probe detected; exact flash was not proven' }, + 'led-blink': { level: 'blocked', raw: 'no probe detected; exact flash was not proven' }, + }); + + // A twin bundle that claims physical evidence must never be accepted. + await record('contaminated/twin', 'twin', shaA, { + heartbeat: { level: 'hardware_observed' }, 'led-blink': { level: 'model_observed' }, + }); + + // A desk bundle bound to a different firmware than the twin ran. + await record('mismatch/desk', 'desk', shaB, { + heartbeat: { level: 'hardware_observed' }, 'led-blink': { level: 'hardware_observed' }, + }); + + process.stdout.write(`artifact A ${shaA}\nartifact B ${shaB}\n`); +} + +await main(); From 0469dc12ff19897c6a7fcdb33c77422cf39cb0ca Mon Sep 17 00:00:00 2001 From: Andrii Shylenko <14119286+w1ne@users.noreply.github.com> Date: Thu, 20 Aug 2026 16:08:59 +0200 Subject: [PATCH 2/2] docs: keep evidence grade names in VERIFY.md, not USAGE.md check-public-package.sh forbids the literal evidence statuses outside docs/VERIFY.md and config/AGENTS.md, and the diff documentation had put model_verified and hardware_observed straight into USAGE.md. USAGE.md now describes the two sides as disjoint model and physical-evidence grades and points at VERIFY.md; VERIFY.md gains the hardware diff section with the exact grade names, the refusal rules, and the verdict-to-exit-code binding. --- docs/USAGE.md | 17 +++++++++-------- docs/VERIFY.md | 26 ++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 8 deletions(-) diff --git a/docs/USAGE.md b/docs/USAGE.md index b37ab45..6fbd476 100644 --- a/docs/USAGE.md +++ b/docs/USAGE.md @@ -159,11 +159,12 @@ accepted as evidence for its side. | `disagree` | `3` | The twin and the board decided a shared behavior differently. This is a first-class published result, not an error. | | `desk-unavailable` | `4` | No desk bundle, or the desk bundle recorded no physical evidence (for example no probe was detected). Never a silent pass. | | `twin-unavailable` | `5` | No twin bundle, or the twin bundle decided nothing. | -| `invalid` | `2` | The comparison was refused: unequal artifact digests, a twin bundle claiming `hardware_observed`, no shared behavior, or a CLI usage error. | - -Each side publishes only its own grade. The twin side can reach -`model_verified` (from `model_observed` or `surrogate_model_observed`); the desk -side can reach `hardware_observed`. `hardware diff` has no code path that copies -a level or a pass from one side to the other, so a hardware green is never -upgraded to a twin green or the reverse. Both sides must be bound to the same -artifact digest; a mismatch is `invalid`, never agreement. +| `invalid` | `2` | The comparison was refused: unequal artifact digests, a twin bundle claiming a physical-evidence grade, no shared behavior, or a CLI usage error. | + +Each side publishes only its own grade, and the grades are disjoint: a model +grade is reachable only from a twin run, a physical-evidence grade only from a +board. `hardware diff` has no code path that copies a level or a pass from one +side to the other, so a hardware green is never upgraded to a twin green or the +reverse. Both sides must be bound to the same artifact digest; a mismatch is +`invalid`, never agreement. The grade names themselves, and what each one +requires, are defined in [VERIFY.md](VERIFY.md). diff --git a/docs/VERIFY.md b/docs/VERIFY.md index 00d9e3f..38b400f 100644 --- a/docs/VERIFY.md +++ b/docs/VERIFY.md @@ -115,3 +115,29 @@ They cannot represent arbitrary twin steps, fix serial capture at 115200 baud, and preserve prebuilt artifacts as imported rather than compiled. Migrate any workflow needing other providers, observations, baud rates, or behavior-level claims to the checked-in generic profile. + +## Comparing the twin against a desk board + +`hardware diff` takes one firmware artifact and two authenticated evidence +bundles — one from the twin, one from a physical board — and publishes whether +they agree. A disagreement is a first-class result, not an error. + +The two sides use disjoint grades and neither is ever converted into the other: + +- The twin side reaches `model_verified`, from `model_observed` or + `surrogate_model_observed`. It can never record `hardware_observed`; a twin + bundle that claims it is rejected as `invalid`. +- The desk side reaches `hardware_observed`, which requires exact flash plus + independent evidence for the configured behavior. A desk record carrying a + model grade is inconclusive desk evidence, never a desk pass. +- `compiled` on the twin side is inconclusive, not behavior evidence. + +Each side is summarized from its own bundle alone, so there is no expression +that can upgrade a hardware green to a twin green or the reverse. Both bundles +must bind to the same artifact digest; a mismatch is refused rather than +smoothed into agreement, and `agree` requires at least one behavior that both +sides actually decided. + +Verdicts bind to exit codes: `agree` 0, `invalid` 2, `disagree` 3, +`desk-unavailable` 4, `twin-unavailable` 5. A failed invocation exits 2, so it +can never be mistaken for a disagreement.