From c44d24df00044e8bd0df426881b71e44391b1c5b Mon Sep 17 00:00:00 2001 From: Andrii Shylenko <14119286+w1ne@users.noreply.github.com> Date: Sat, 15 Aug 2026 20:15:28 +0200 Subject: [PATCH 1/6] fix: accept proven bounded twin verification --- tests/develop-agent-oracle-test.py | 6 ++++++ tests/develop-agent-oracle.py | 4 +++- tests/fixtures/develop-agent/empty-refresh-valid.jsonl | 2 +- .../develop-agent/opencode-max-steps-verified.jsonl | 5 +++++ tests/fixtures/develop-agent/opencode-valid.jsonl | 2 +- 5 files changed, 16 insertions(+), 3 deletions(-) create mode 100644 tests/fixtures/develop-agent/opencode-max-steps-verified.jsonl diff --git a/tests/develop-agent-oracle-test.py b/tests/develop-agent-oracle-test.py index 8c20ed6..4e91b1d 100755 --- a/tests/develop-agent-oracle-test.py +++ b/tests/develop-agent-oracle-test.py @@ -30,6 +30,12 @@ def run(name): assert opencode_bundle["final_claim"] == "model_verified" assert opencode_bundle["source_citations"] == ["catalog:board:esp32-c3-supermini"] +max_steps_verified = subprocess.run( + [sys.executable, str(ORACLE), "validate", str(FIX / "opencode-max-steps-verified.jsonl"), "existing-stm32f103"], + text=True, capture_output=True, +) +assert max_steps_verified.returncode == 0, max_steps_verified.stderr + refreshed = run("empty-refresh-valid.jsonl") assert refreshed.returncode == 0, refreshed.stderr diff --git a/tests/develop-agent-oracle.py b/tests/develop-agent-oracle.py index 35d7fcd..6ac2846 100755 --- a/tests/develop-agent-oracle.py +++ b/tests/develop-agent-oracle.py @@ -240,7 +240,9 @@ def phase_outcome(event: dict, phase: str) -> bool | None: clauses = result.get("oracle_results") if ( result.get("proven") is True - and result.get("stop_reason") == "assertions_passed" + and result.get("gaps") == [] + and result.get("run_status") in (None, "pass") + and result.get("stop_reason") in {"assertions_passed", "max_steps"} and isinstance(clauses, list) and clauses and all(isinstance(clause, dict) and clause.get("passed") is True for clause in clauses) diff --git a/tests/fixtures/develop-agent/empty-refresh-valid.jsonl b/tests/fixtures/develop-agent/empty-refresh-valid.jsonl index 88b18c5..8af7af6 100644 --- a/tests/fixtures/develop-agent/empty-refresh-valid.jsonl +++ b/tests/fixtures/develop-agent/empty-refresh-valid.jsonl @@ -2,5 +2,5 @@ {"type":"tool_use","part":{"id":"r2","tool":"labwired_describe","state":{"status":"completed","input":{"id":"esp32-c3-supermini"},"output":"{\"board\":\"esp32-c3-supermini\",\"chip\":\"esp32c3\"}"}}} {"type":"tool_use","part":{"id":"r3","tool":"labwired_context","state":{"status":"completed","input":{"pack":{"board":"esp32-c3-supermini","mcu":"esp32c3"}},"output":"{\"ok\":true,\"design_context_ok\":true,\"board\":\"esp32-c3-supermini\",\"mcu\":\"esp32c3\"}"}}} {"type":"tool_use","part":{"id":"r4","tool":"labwired_compile","state":{"status":"completed","input":{},"output":"{\"ok\":true,\"runnable\":true,\"flash_image_refs\":[{\"offset\":0,\"ref\":\"sha256:firmware\"}]}"}}} -{"type":"tool_use","part":{"id":"r5","tool":"labwired_verify","state":{"status":"completed","input":{},"output":"{\"ok\":true,\"status\":\"model_verified\",\"proven\":true,\"stop_reason\":\"assertions_passed\",\"oracle_results\":[{\"passed\":true,\"detail\":\"serial contains alive\"}]}"}}} +{"type":"tool_use","part":{"id":"r5","tool":"labwired_verify","state":{"status":"completed","input":{},"output":"{\"ok\":true,\"status\":\"model_verified\",\"proven\":true,\"gaps\":[],\"run_status\":\"pass\",\"stop_reason\":\"assertions_passed\",\"oracle_results\":[{\"passed\":true,\"detail\":\"serial contains alive\"}]}"}}} {"type":"text","part":{"id":"r6","text":"{\"claim\":\"model_verified\"}"}} diff --git a/tests/fixtures/develop-agent/opencode-max-steps-verified.jsonl b/tests/fixtures/develop-agent/opencode-max-steps-verified.jsonl new file mode 100644 index 0000000..e23c68e --- /dev/null +++ b/tests/fixtures/develop-agent/opencode-max-steps-verified.jsonl @@ -0,0 +1,5 @@ +{"type":"tool_use","part":{"id":"e1","tool":"labwired_context","state":{"status":"completed","output":"{\"ok\":true,\"design_context_ok\":true,\"board\":\"stm32f103-blinky\",\"mcu\":\"stm32f103\"}"}}} +{"type":"tool_use","part":{"id":"e2","tool":"labwired_describe","state":{"status":"completed","output":"{\"board\":\"stm32f103-blinky\"}"}}} +{"type":"tool_use","part":{"id":"e3","tool":"labwired_compile","state":{"status":"completed","output":"{\"ok\":true,\"firmware_ref\":\"sha256:firmware-fixture\",\"runnable\":true}"}}} +{"type":"tool_use","part":{"id":"e4","tool":"labwired_verify","state":{"status":"completed","output":"{\"ok\":true,\"status\":\"model_verified\",\"proven\":true,\"gaps\":[],\"run_status\":\"pass\",\"stop_reason\":\"max_steps\",\"oracle_results\":[{\"kind\":\"gpio\",\"passed\":true,\"detail\":\"PC13 toggled (gpio evidence)\"}]}"}}} +{"type":"text","part":{"text":"{\"claim\":\"model_verified\",\"hardware_sensitive_facts\":[{\"fact\":\"PC13 is the board heartbeat LED pin\",\"source\":\"catalog:board:stm32f103-blinky\"}]}"}} diff --git a/tests/fixtures/develop-agent/opencode-valid.jsonl b/tests/fixtures/develop-agent/opencode-valid.jsonl index 6b8b8c6..d606284 100644 --- a/tests/fixtures/develop-agent/opencode-valid.jsonl +++ b/tests/fixtures/develop-agent/opencode-valid.jsonl @@ -1,5 +1,5 @@ {"type":"tool_use","part":{"id":"oc1","tool":"labwired_context","state":{"status":"completed","input":{},"output":"{\"ok\":true,\"mode\":\"design_only\",\"design_context_ok\":true,\"twin_buildable\":false,\"board\":\"esp32-c3-supermini\",\"mcu\":\"esp32c3\"}"}}} {"type":"tool_use","part":{"id":"oc2","tool":"labwired_describe","state":{"status":"completed","input":{"id":"esp32-c3-supermini"},"output":"{\"board\":\"esp32-c3-supermini\",\"chip\":\"esp32c3\",\"description\":\"Built-in user LED on GPIO8 (active-low)\"}"}}} {"type":"tool_use","part":{"id":"oc3","tool":"labwired_compile","state":{"status":"completed","input":{"project":"."},"output":"{\"ok\":true,\"runnable\":true,\"flash_image_refs\":[{\"offset\":0,\"ref\":\"sha256:firmware-fixture\",\"blob_path\":\"/v1/blobs/sha256/firmware-fixture\"}]}"}}} -{"type":"tool_use","part":{"id":"oc4","tool":"labwired_verify","state":{"status":"completed","input":{"firmware":"sha256:firmware-fixture"},"output":"{\"ok\":true,\"status\":\"model_verified\",\"proven\":true,\"stop_reason\":\"assertions_passed\",\"oracle_results\":[{\"passed\":true,\"detail\":\"serial contains alive\"}]}"}}} +{"type":"tool_use","part":{"id":"oc4","tool":"labwired_verify","state":{"status":"completed","input":{"firmware":"sha256:firmware-fixture"},"output":"{\"ok\":true,\"status\":\"model_verified\",\"proven\":true,\"gaps\":[],\"run_status\":\"pass\",\"stop_reason\":\"assertions_passed\",\"oracle_results\":[{\"passed\":true,\"detail\":\"serial contains alive\"}]}"}}} {"type":"text","part":{"id":"oc5","type":"text","text":"Final claim: model_verified"}} From b07c3451386ae619b969f82988d0b562018ad858 Mon Sep 17 00:00:00 2001 From: Andrii Shylenko <14119286+w1ne@users.noreply.github.com> Date: Sat, 15 Aug 2026 20:29:55 +0200 Subject: [PATCH 2/6] fix: make compile recovery evidence reliable --- tests/develop-agent-oracle-test.py | 6 ++++++ tests/develop-agent-oracle.py | 7 +++++++ tests/develop-agent-wiring-test.py | 7 +++++++ .../develop-agent/opencode-recovery-edit-valid.jsonl | 7 +++++++ tests/fixtures/develop-agent/prompts.json | 2 +- 5 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 tests/fixtures/develop-agent/opencode-recovery-edit-valid.jsonl diff --git a/tests/develop-agent-oracle-test.py b/tests/develop-agent-oracle-test.py index 4e91b1d..eb64d6c 100755 --- a/tests/develop-agent-oracle-test.py +++ b/tests/develop-agent-oracle-test.py @@ -117,6 +117,12 @@ def run(name): assert ambiguous.returncode != 0 assert "explicit failed compile" in ambiguous.stderr.lower(), ambiguous.stderr +opencode_recovery = subprocess.run( + [sys.executable, str(ORACLE), "validate", str(FIX / "opencode-recovery-edit-valid.jsonl"), "compile-recovery-esp32c3"], + text=True, capture_output=True, +) +assert opencode_recovery.returncode == 0, opencode_recovery.stderr + secrets = run("secrets.jsonl") assert secrets.returncode == 0, secrets.stderr for forbidden in ("secret-token-value", "signed-secret", "person@example.com", "Bearer"): diff --git a/tests/develop-agent-oracle.py b/tests/develop-agent-oracle.py index 6ac2846..8f0bd2d 100755 --- a/tests/develop-agent-oracle.py +++ b/tests/develop-agent-oracle.py @@ -202,6 +202,13 @@ def typed_record(result: dict, key: str, expected_type: str) -> dict | None: def phase_outcome(event: dict, phase: str) -> bool | None: + # OpenCode's built-in edit tool returns a fixed success sentence rather + # than a JSON domain envelope. Accept only that exact canonical response; + # arbitrary prose or model-supplied command text remains non-evidence. + if phase == "edit" and tool_name(event).lower() == "edit" and outcome(event) is True: + payloads = returned_payloads(event) + if payloads == ["Edit applied successfully."]: + return True domain = domain_outcome(event) if domain is False: return False diff --git a/tests/develop-agent-wiring-test.py b/tests/develop-agent-wiring-test.py index 2912fd3..1337887 100755 --- a/tests/develop-agent-wiring-test.py +++ b/tests/develop-agent-wiring-test.py @@ -34,6 +34,13 @@ assert 'source must be exactly catalog:board:esp32-c3-supermini' in prompts["greenfield-esp32c3"] assert 'source must be exactly catalog:board:stm32f103-blinky' in prompts["existing-stm32f103"] assert 'source must be exactly catalog:board:esp32-c3-supermini' in prompts["compile-recovery-esp32c3"] +recovery_prompt = prompts["compile-recovery-esp32c3"] +for required in ( + "The project already contains the deliberate error in src/main.cpp; do not overwrite or replace it before the first compile.", + "After the failed compile, use the edit tool on src/main.cpp for one focused repair", + "compile the exact repaired file content", +): + assert required in recovery_prompt assert 'source must be exactly catalog:board:esp32-c3-supermini' in prompts["partial-led-wifi"] for name in ("greenfield-esp32c3", "existing-stm32f103", "compile-recovery-esp32c3", "partial-led-wifi"): assert "GPIO oracle state must be toggled" in prompts[name] diff --git a/tests/fixtures/develop-agent/opencode-recovery-edit-valid.jsonl b/tests/fixtures/develop-agent/opencode-recovery-edit-valid.jsonl new file mode 100644 index 0000000..1a277ff --- /dev/null +++ b/tests/fixtures/develop-agent/opencode-recovery-edit-valid.jsonl @@ -0,0 +1,7 @@ +{"type":"tool_use","part":{"id":"r1","tool":"labwired_context","state":{"status":"completed","output":"{\"ok\":true,\"design_context_ok\":true,\"board\":\"esp32-c3-supermini\",\"mcu\":\"esp32c3\"}"}}} +{"type":"tool_use","part":{"id":"r2","tool":"labwired_describe","state":{"status":"completed","output":"{\"board\":\"esp32-c3-supermini\"}"}}} +{"type":"tool_use","part":{"id":"r3","tool":"labwired_compile","state":{"status":"error","error":"compiler rejected source"}}} +{"type":"tool_use","part":{"id":"r4","tool":"edit","state":{"status":"completed","output":"Edit applied successfully."}}} +{"type":"tool_use","part":{"id":"r5","tool":"labwired_compile","state":{"status":"completed","output":"{\"ok\":true,\"runnable\":true,\"firmware_ref\":\"sha256:repaired-firmware\"}"}}} +{"type":"tool_use","part":{"id":"r6","tool":"labwired_verify","state":{"status":"completed","output":"{\"ok\":true,\"status\":\"model_verified\",\"proven\":true,\"gaps\":[],\"run_status\":\"pass\",\"stop_reason\":\"assertions_passed\",\"oracle_results\":[{\"kind\":\"serial\",\"passed\":true,\"detail\":\"serial contains alive\"},{\"kind\":\"gpio\",\"passed\":true,\"detail\":\"GPIO8 toggled\"}]}"}}} +{"type":"text","part":{"text":"{\"claim\":\"model_verified\",\"hardware_sensitive_facts\":[{\"fact\":\"GPIO8 is the active-low board LED pin\",\"source\":\"catalog:board:esp32-c3-supermini\"}]}"}} diff --git a/tests/fixtures/develop-agent/prompts.json b/tests/fixtures/develop-agent/prompts.json index afa4a22..8c78a4d 100644 --- a/tests/fixtures/develop-agent/prompts.json +++ b/tests/fixtures/develop-agent/prompts.json @@ -1,7 +1,7 @@ { "greenfield-esp32c3": "Create PlatformIO Arduino firmware for ESP32-C3 DevKitM-1 that blinks the configured LED once per second and prints `alive` over serial. If the first labwired_context returns empty_context, resolve the board with list/describe, then call labwired_context again with pack.board and pack.mcu and require ok true before compiling. Use labwired_compile for the compile evidence, not a shell compile or project-local command. Call labwired_describe for esp32-c3-supermini; the final source must be exactly catalog:board:esp32-c3-supermini. Verify both behaviors by calling labwired_verify directly with the firmware and a combined serial plus GPIO oracle; do not start with labwired_run or reuse its run_id. The GPIO oracle state must be toggled, never high or low, because a static level is not a blink. Set output summary and never request full or peripherals. Do not delegate or start a subagent; keep every tool call and all evidence in this session. Return only this JSON shape and nothing else: {\"claim\":\"model_verified\",\"hardware_sensitive_facts\":[{\"fact\":\"GPIO8 is the active-low board LED pin\",\"source\":\"catalog:board:esp32-c3-supermini\"}]}.", "existing-stm32f103": "Add a one-second heartbeat without restructuring this STM32F103 project. If the first labwired_context returns empty_context, resolve the board with list/describe, then call labwired_context again with pack.board and pack.mcu and require ok true before compiling. Use labwired_compile for the compile evidence, not a shell compile or project-local command. Call labwired_describe for stm32f103-blinky; the final source must be exactly catalog:board:stm32f103-blinky. Preserve layout. Call labwired_verify directly with the firmware, a board/MCU-only diagram with no external LED part or wires, a PC13 GPIO oracle, max_steps 50000000, and output summary; do not start with labwired_run or reuse its run_id. The GPIO oracle state must be toggled, never high or low, because a static level is not a heartbeat. Do not delegate or start a subagent; keep every tool call and all evidence in this session. Return only this JSON shape and nothing else: {\"claim\":\"model_verified\",\"hardware_sensitive_facts\":[{\"fact\":\"PC13 is the board heartbeat LED pin\",\"source\":\"catalog:board:stm32f103-blinky\"}]}.", - "compile-recovery-esp32c3": "Create PlatformIO Arduino firmware for ESP32-C3 DevKitM-1 that blinks the configured LED once per second and prints `alive` over serial. If the first labwired_context returns empty_context, resolve the board with list/describe, then call labwired_context again with pack.board and pack.mcu and require ok true before compiling. Use labwired_compile for every compile attempt, not a shell compile or project-local command. Keep the deliberate compiler error initially, use its diagnostic for one focused repair, then compile successfully. Call labwired_describe for esp32-c3-supermini; the final source must be exactly catalog:board:esp32-c3-supermini. Call labwired_verify directly with the repaired firmware and a combined serial plus GPIO oracle; do not start with labwired_run or reuse its run_id. The GPIO oracle state must be toggled, never high or low, because a static level is not a blink. Set output summary and never request full or peripherals. Do not delegate or start a subagent; keep every tool call and all evidence in this session. Return only this JSON shape and nothing else: {\"claim\":\"model_verified\",\"hardware_sensitive_facts\":[{\"fact\":\"GPIO8 is the active-low board LED pin\",\"source\":\"catalog:board:esp32-c3-supermini\"}]}.", + "compile-recovery-esp32c3": "Create PlatformIO Arduino firmware for ESP32-C3 DevKitM-1 that blinks the configured LED once per second and prints `alive` over serial. The project already contains the deliberate error in src/main.cpp; do not overwrite or replace it before the first compile. If the first labwired_context returns empty_context, resolve the board with list/describe, then call labwired_context again with pack.board and pack.mcu and require ok true before compiling. Read src/main.cpp. Use labwired_compile on that exact broken source for the first compile attempt (use Arduino entryPath src/main.ino when submitting it to the hosted compiler), not a shell compile or project-local command. After the failed compile, use the edit tool on src/main.cpp for one focused repair based on its diagnostic, then compile the exact repaired file content successfully with labwired_compile. Call labwired_describe for esp32-c3-supermini; the final source must be exactly catalog:board:esp32-c3-supermini. Call labwired_verify directly with the repaired firmware and a combined serial plus GPIO oracle; do not start with labwired_run or reuse its run_id. The GPIO oracle state must be toggled, never high or low, because a static level is not a blink. Set output summary and never request full or peripherals. Do not delegate or start a subagent; keep every tool call and all evidence in this session. Return only this JSON shape and nothing else: {\"claim\":\"model_verified\",\"hardware_sensitive_facts\":[{\"fact\":\"GPIO8 is the active-low board LED pin\",\"source\":\"catalog:board:esp32-c3-supermini\"}]}.", "partial-led-wifi": "Create ESP32-C3 firmware that blinks the LED once per second and associates with Wi-Fi. If the first labwired_context returns empty_context, resolve the board with list/describe, then call labwired_context again with pack.board and pack.mcu and require ok true before compiling. Use labwired_compile for the compile evidence, not a shell compile or project-local command. Call labwired_describe for esp32-c3-supermini; the final source must be exactly catalog:board:esp32-c3-supermini. Call labwired_verify directly with the firmware and GPIO oracle; do not start with labwired_run or reuse its run_id. The GPIO oracle state must be toggled, never high or low, because a static level is not a blink. Set output summary and never request full or peripherals. Report Wi-Fi as an uncovered gap unless association is independently checked, so do not claim model_verified. Do not delegate or start a subagent; keep every tool call and all evidence in this session. Return only this JSON shape and nothing else: {\"claim\":\"partially_verified\",\"hardware_sensitive_facts\":[{\"fact\":\"GPIO8 is the active-low board LED pin\",\"source\":\"catalog:board:esp32-c3-supermini\"}]}.", "unsupported-custom-board": "Build this unsupported custom-board firmware. If the first labwired_context returns empty_context, resolve available board information with list/describe, then call labwired_context again with pack.board and pack.mcu and require ok true before compiling. Use labwired_compile for the compile attempt, not a shell compile or project-local command. Ground one available hardware fact in a returned part, datasheet, SDK, SVD, schematic, netlist, or project citation. Attempt the available verification path, state that physical confirmation is still required, and never claim model or hardware verification without oracle evidence. Do not delegate or start a subagent; keep every tool call and all evidence in this session. Return only a JSON object with claim exactly compiled_only and one hardware_sensitive_facts fact/source object whose source is an exact returned citation token." } From 06a1d0ded400d7f19b8935969241246d6fcc8a61 Mon Sep 17 00:00:00 2001 From: Andrii Shylenko <14119286+w1ne@users.noreply.github.com> Date: Sat, 15 Aug 2026 20:53:19 +0200 Subject: [PATCH 3/6] fix: make partial and unsupported workflows deterministic --- tests/develop-agent-wiring-test.py | 18 ++++++++++++++++++ tests/fixtures/develop-agent/prompts.json | 4 ++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/tests/develop-agent-wiring-test.py b/tests/develop-agent-wiring-test.py index 1337887..bacdf3c 100755 --- a/tests/develop-agent-wiring-test.py +++ b/tests/develop-agent-wiring-test.py @@ -41,6 +41,24 @@ "compile the exact repaired file content", ): assert required in recovery_prompt +partial_prompt = prompts["partial-led-wifi"] +for required in ( + "upload the complete project with labwired_put_source", + "compile its returned source_tree_ref", + "target esp32-c3-supermini and the returned flash_image_refs", + "serial contains `Connecting to Wi-Fi` plus GPIO8 toggled", + 'refresh context with exactly pack.board `esp32-c3-supermini` and pack.mcu `esp32c3`', + "do not include false context flags", +): + assert required in partial_prompt +unsupported_prompt = prompts["unsupported-custom-board"] +for required in ( + "nucleo-f401re only as a compiler surrogate", + "attempt labwired_verify with target custom-board", + "Physical confirmation is still required", + '"hardware_sensitive_facts":[{"fact":"PA5 is the surrogate board LED pin","source":"catalog:board:nucleo-f401re"}]', +): + assert required in unsupported_prompt assert 'source must be exactly catalog:board:esp32-c3-supermini' in prompts["partial-led-wifi"] for name in ("greenfield-esp32c3", "existing-stm32f103", "compile-recovery-esp32c3", "partial-led-wifi"): assert "GPIO oracle state must be toggled" in prompts[name] diff --git a/tests/fixtures/develop-agent/prompts.json b/tests/fixtures/develop-agent/prompts.json index 8c78a4d..43ba45b 100644 --- a/tests/fixtures/develop-agent/prompts.json +++ b/tests/fixtures/develop-agent/prompts.json @@ -2,6 +2,6 @@ "greenfield-esp32c3": "Create PlatformIO Arduino firmware for ESP32-C3 DevKitM-1 that blinks the configured LED once per second and prints `alive` over serial. If the first labwired_context returns empty_context, resolve the board with list/describe, then call labwired_context again with pack.board and pack.mcu and require ok true before compiling. Use labwired_compile for the compile evidence, not a shell compile or project-local command. Call labwired_describe for esp32-c3-supermini; the final source must be exactly catalog:board:esp32-c3-supermini. Verify both behaviors by calling labwired_verify directly with the firmware and a combined serial plus GPIO oracle; do not start with labwired_run or reuse its run_id. The GPIO oracle state must be toggled, never high or low, because a static level is not a blink. Set output summary and never request full or peripherals. Do not delegate or start a subagent; keep every tool call and all evidence in this session. Return only this JSON shape and nothing else: {\"claim\":\"model_verified\",\"hardware_sensitive_facts\":[{\"fact\":\"GPIO8 is the active-low board LED pin\",\"source\":\"catalog:board:esp32-c3-supermini\"}]}.", "existing-stm32f103": "Add a one-second heartbeat without restructuring this STM32F103 project. If the first labwired_context returns empty_context, resolve the board with list/describe, then call labwired_context again with pack.board and pack.mcu and require ok true before compiling. Use labwired_compile for the compile evidence, not a shell compile or project-local command. Call labwired_describe for stm32f103-blinky; the final source must be exactly catalog:board:stm32f103-blinky. Preserve layout. Call labwired_verify directly with the firmware, a board/MCU-only diagram with no external LED part or wires, a PC13 GPIO oracle, max_steps 50000000, and output summary; do not start with labwired_run or reuse its run_id. The GPIO oracle state must be toggled, never high or low, because a static level is not a heartbeat. Do not delegate or start a subagent; keep every tool call and all evidence in this session. Return only this JSON shape and nothing else: {\"claim\":\"model_verified\",\"hardware_sensitive_facts\":[{\"fact\":\"PC13 is the board heartbeat LED pin\",\"source\":\"catalog:board:stm32f103-blinky\"}]}.", "compile-recovery-esp32c3": "Create PlatformIO Arduino firmware for ESP32-C3 DevKitM-1 that blinks the configured LED once per second and prints `alive` over serial. The project already contains the deliberate error in src/main.cpp; do not overwrite or replace it before the first compile. If the first labwired_context returns empty_context, resolve the board with list/describe, then call labwired_context again with pack.board and pack.mcu and require ok true before compiling. Read src/main.cpp. Use labwired_compile on that exact broken source for the first compile attempt (use Arduino entryPath src/main.ino when submitting it to the hosted compiler), not a shell compile or project-local command. After the failed compile, use the edit tool on src/main.cpp for one focused repair based on its diagnostic, then compile the exact repaired file content successfully with labwired_compile. Call labwired_describe for esp32-c3-supermini; the final source must be exactly catalog:board:esp32-c3-supermini. Call labwired_verify directly with the repaired firmware and a combined serial plus GPIO oracle; do not start with labwired_run or reuse its run_id. The GPIO oracle state must be toggled, never high or low, because a static level is not a blink. Set output summary and never request full or peripherals. Do not delegate or start a subagent; keep every tool call and all evidence in this session. Return only this JSON shape and nothing else: {\"claim\":\"model_verified\",\"hardware_sensitive_facts\":[{\"fact\":\"GPIO8 is the active-low board LED pin\",\"source\":\"catalog:board:esp32-c3-supermini\"}]}.", - "partial-led-wifi": "Create ESP32-C3 firmware that blinks the LED once per second and associates with Wi-Fi. If the first labwired_context returns empty_context, resolve the board with list/describe, then call labwired_context again with pack.board and pack.mcu and require ok true before compiling. Use labwired_compile for the compile evidence, not a shell compile or project-local command. Call labwired_describe for esp32-c3-supermini; the final source must be exactly catalog:board:esp32-c3-supermini. Call labwired_verify directly with the firmware and GPIO oracle; do not start with labwired_run or reuse its run_id. The GPIO oracle state must be toggled, never high or low, because a static level is not a blink. Set output summary and never request full or peripherals. Report Wi-Fi as an uncovered gap unless association is independently checked, so do not claim model_verified. Do not delegate or start a subagent; keep every tool call and all evidence in this session. Return only this JSON shape and nothing else: {\"claim\":\"partially_verified\",\"hardware_sensitive_facts\":[{\"fact\":\"GPIO8 is the active-low board LED pin\",\"source\":\"catalog:board:esp32-c3-supermini\"}]}.", - "unsupported-custom-board": "Build this unsupported custom-board firmware. If the first labwired_context returns empty_context, resolve available board information with list/describe, then call labwired_context again with pack.board and pack.mcu and require ok true before compiling. Use labwired_compile for the compile attempt, not a shell compile or project-local command. Ground one available hardware fact in a returned part, datasheet, SDK, SVD, schematic, netlist, or project citation. Attempt the available verification path, state that physical confirmation is still required, and never claim model or hardware verification without oracle evidence. Do not delegate or start a subagent; keep every tool call and all evidence in this session. Return only a JSON object with claim exactly compiled_only and one hardware_sensitive_facts fact/source object whose source is an exact returned citation token." + "partial-led-wifi": "Create ESP32-C3 firmware that blinks the LED once per second and starts Wi-Fi association without blocking the LED loop. Print `Connecting to Wi-Fi` before starting association. If the first labwired_context returns empty_context, call labwired_describe for esp32-c3-supermini, then call labwired_context again to refresh context with exactly pack.board `esp32-c3-supermini` and pack.mcu `esp32c3`; do not include false context flags, and require ok true before compiling. Use labwired_compile for the compile evidence, not a shell compile or project-local command: upload the complete project with labwired_put_source, then compile its returned source_tree_ref so WiFi.h dependencies are resolved. The final source must be exactly catalog:board:esp32-c3-supermini. Call labwired_verify directly using target esp32-c3-supermini and the returned flash_image_refs, a board/MCU-only diagram, and a combined oracle requiring serial contains `Connecting to Wi-Fi` plus GPIO8 toggled; do not start with labwired_run or reuse its run_id. The GPIO oracle state must be toggled, never high or low, because a static level is not a blink. Set output summary and never request full or peripherals. Report Wi-Fi association as an uncovered gap because only its start message is checked, so do not claim model_verified. Do not delegate or start a subagent; keep every tool call and all evidence in this session. Return only this JSON shape and nothing else: {\"claim\":\"partially_verified\",\"hardware_sensitive_facts\":[{\"fact\":\"GPIO8 is the active-low board LED pin\",\"source\":\"catalog:board:esp32-c3-supermini\"}]}.", + "unsupported-custom-board": "Build this unsupported custom-board firmware without pretending the custom hardware has a twin. If the first labwired_context returns empty_context, call labwired_describe for nucleo-f401re, then call labwired_context again with exactly pack.board nucleo-f401re and pack.mcu stm32f401 and require ok true before compiling. Ground PA5 from that describe result; the final source must be exactly catalog:board:nucleo-f401re. Use labwired_compile with nucleo-f401re only as a compiler surrogate, not a shell compile or project-local command; this proves only that the portable firmware builds, not that it works on the custom board. Then attempt labwired_verify with target custom-board so the unavailable twin path is recorded; do not verify the surrogate target. Physical confirmation is still required. Never claim model_verified or hardware_observed. Do not delegate or start a subagent; keep every tool call and all evidence in this session. Return only this JSON shape and nothing else: {\"claim\":\"compiled_only\",\"hardware_sensitive_facts\":[{\"fact\":\"PA5 is the surrogate board LED pin\",\"source\":\"catalog:board:nucleo-f401re\"}]}." } From bf5234ec80ad5dde5b6b15ba40c72fb6fd07b0b9 Mon Sep 17 00:00:00 2001 From: Andrii Shylenko <14119286+w1ne@users.noreply.github.com> Date: Sat, 15 Aug 2026 21:06:56 +0200 Subject: [PATCH 4/6] fix: keep context refresh inputs clean --- skills/develop/SKILL.md | 2 +- tests/develop-agent-wiring-test.py | 7 +++++++ tests/develop-skill.sh | 1 + tests/fixtures/develop-agent/prompts.json | 2 +- 4 files changed, 10 insertions(+), 2 deletions(-) diff --git a/skills/develop/SKILL.md b/skills/develop/SKILL.md index 88ef0a1..e635086 100644 --- a/skills/develop/SKILL.md +++ b/skills/develop/SKILL.md @@ -16,7 +16,7 @@ Follow this loop: inspect or scaffold → ground → edit → compile → twin c Rules: -- Call `labwired_context` first. On `empty_context`, use `labwired_list` + `labwired_describe`, then call `labwired_context` again with `pack.board` and `pack.mcu` from that returned catalog board before compiling; for an external circuit, pass the pack returned by `labwired_import`. This second/refreshed `labwired_context` is mandatory and must succeed with `ok: true` and `design_context_ok: true` before any compile. If refreshed context has `ok: false` or `design_context_ok: false`, do not compile. Cite described boards as exact `catalog:board:` sources (and components as `catalog:component:`). Prefer `labwired_part`, `labwired_datasheet`, or `labwired_search` for other hardware facts. If those knowledge tools are unavailable, grounded existing project files, SDK headers, SVD files, and schematics or netlists are valid fallback sources; cite the source used. Label missing-source deductions as inferred or as a gap, and never invent pins, peripherals, addresses, clocks, timing, or registers. +- Call `labwired_context` first. On `empty_context`, use `labwired_list` + `labwired_describe`, then call `labwired_context` again with `pack.board` and `pack.mcu` from that returned catalog board before compiling; for an external circuit, pass the pack returned by `labwired_import`. A refreshed pack contains only resolved inputs—never copy stale `false`, negative-status, or other output fields into it. This second/refreshed `labwired_context` is mandatory and must succeed with `ok: true` and `design_context_ok: true` before any compile. If refreshed context has `ok: false` or `design_context_ok: false`, do not compile. Cite described boards as exact `catalog:board:` sources (and components as `catalog:component:`). Prefer `labwired_part`, `labwired_datasheet`, or `labwired_search` for other hardware facts. If those knowledge tools are unavailable, grounded existing project files, SDK headers, SVD files, and schematics or netlists are valid fallback sources; cite the source used. Label missing-source deductions as inferred or as a gap, and never invent pins, peripherals, addresses, clocks, timing, or registers. - Preserve an existing project's structure. For greenfield work, create the smallest conventional project that satisfies the request. - Compile with `labwired_compile` or the project's existing compile command. - Prefer a reviewed, checked-in safe `.labwired/hardware.json` hardware profile diff --git a/tests/develop-agent-wiring-test.py b/tests/develop-agent-wiring-test.py index bacdf3c..795566b 100755 --- a/tests/develop-agent-wiring-test.py +++ b/tests/develop-agent-wiring-test.py @@ -34,6 +34,13 @@ assert 'source must be exactly catalog:board:esp32-c3-supermini' in prompts["greenfield-esp32c3"] assert 'source must be exactly catalog:board:stm32f103-blinky' in prompts["existing-stm32f103"] assert 'source must be exactly catalog:board:esp32-c3-supermini' in prompts["compile-recovery-esp32c3"] +stm32_prompt = prompts["existing-stm32f103"] +for required in ( + "edit only the existing src/main.cpp in place", + "do not add, remove, or rename files", + "leave platformio.ini byte-identical", +): + assert required in stm32_prompt recovery_prompt = prompts["compile-recovery-esp32c3"] for required in ( "The project already contains the deliberate error in src/main.cpp; do not overwrite or replace it before the first compile.", diff --git a/tests/develop-skill.sh b/tests/develop-skill.sh index 9483166..e273fe9 100755 --- a/tests/develop-skill.sh +++ b/tests/develop-skill.sh @@ -44,6 +44,7 @@ fi need 'labwired_context' 'context tool' need 'empty[_ -]?context.*(resolve|import|select|labwired_list|labwired_describe).*(labwired_context.*(again|re-run)|re-run.*labwired_context).*before.*compil' 'empty context must be resolved and refreshed before compile' need 'labwired_context.*pack.*board.*mcu' 'catalog board context is refreshed with an explicit pack' +need '(refresh|refreshed).*pack.*(only|resolved inputs).*(never|do not).*(false|negative|status|output)' 'context refresh excludes stale negative output fields' need '(second|refreshed).*(labwired_context|context).*(must|mandatory).*(succeed|ok|design_context_ok).*before.*compil' 'refreshed context success is a hard compile gate' need '(ok.*false|design_context_ok.*false).*(do not|never|must not).*compil' 'failed context blocks compile' need 'catalog:board:.*returned.*board|returned.*board.*catalog:board:' 'catalog board citations use the canonical returned id' diff --git a/tests/fixtures/develop-agent/prompts.json b/tests/fixtures/develop-agent/prompts.json index 43ba45b..91d8402 100644 --- a/tests/fixtures/develop-agent/prompts.json +++ b/tests/fixtures/develop-agent/prompts.json @@ -1,6 +1,6 @@ { "greenfield-esp32c3": "Create PlatformIO Arduino firmware for ESP32-C3 DevKitM-1 that blinks the configured LED once per second and prints `alive` over serial. If the first labwired_context returns empty_context, resolve the board with list/describe, then call labwired_context again with pack.board and pack.mcu and require ok true before compiling. Use labwired_compile for the compile evidence, not a shell compile or project-local command. Call labwired_describe for esp32-c3-supermini; the final source must be exactly catalog:board:esp32-c3-supermini. Verify both behaviors by calling labwired_verify directly with the firmware and a combined serial plus GPIO oracle; do not start with labwired_run or reuse its run_id. The GPIO oracle state must be toggled, never high or low, because a static level is not a blink. Set output summary and never request full or peripherals. Do not delegate or start a subagent; keep every tool call and all evidence in this session. Return only this JSON shape and nothing else: {\"claim\":\"model_verified\",\"hardware_sensitive_facts\":[{\"fact\":\"GPIO8 is the active-low board LED pin\",\"source\":\"catalog:board:esp32-c3-supermini\"}]}.", - "existing-stm32f103": "Add a one-second heartbeat without restructuring this STM32F103 project. If the first labwired_context returns empty_context, resolve the board with list/describe, then call labwired_context again with pack.board and pack.mcu and require ok true before compiling. Use labwired_compile for the compile evidence, not a shell compile or project-local command. Call labwired_describe for stm32f103-blinky; the final source must be exactly catalog:board:stm32f103-blinky. Preserve layout. Call labwired_verify directly with the firmware, a board/MCU-only diagram with no external LED part or wires, a PC13 GPIO oracle, max_steps 50000000, and output summary; do not start with labwired_run or reuse its run_id. The GPIO oracle state must be toggled, never high or low, because a static level is not a heartbeat. Do not delegate or start a subagent; keep every tool call and all evidence in this session. Return only this JSON shape and nothing else: {\"claim\":\"model_verified\",\"hardware_sensitive_facts\":[{\"fact\":\"PC13 is the board heartbeat LED pin\",\"source\":\"catalog:board:stm32f103-blinky\"}]}.", + "existing-stm32f103": "Add a one-second heartbeat without restructuring this STM32F103 project: read and edit only the existing src/main.cpp in place; do not add, remove, or rename files; leave platformio.ini byte-identical. If the first labwired_context returns empty_context, resolve the board with list/describe, then call labwired_context again with pack.board and pack.mcu and require ok true before compiling. Use labwired_compile for the compile evidence, not a shell compile or project-local command; submit the exact src/main.cpp content with Arduino entryPath src/main.ino without creating that file. Call labwired_describe for stm32f103-blinky; the final source must be exactly catalog:board:stm32f103-blinky. Call labwired_verify directly with the firmware, a board/MCU-only diagram with no external LED part or wires, a PC13 GPIO oracle, max_steps 50000000, and output summary; do not start with labwired_run or reuse its run_id. The GPIO oracle state must be toggled, never high or low, because a static level is not a heartbeat. Do not delegate or start a subagent; keep every tool call and all evidence in this session. Return only this JSON shape and nothing else: {\"claim\":\"model_verified\",\"hardware_sensitive_facts\":[{\"fact\":\"PC13 is the board heartbeat LED pin\",\"source\":\"catalog:board:stm32f103-blinky\"}]}.", "compile-recovery-esp32c3": "Create PlatformIO Arduino firmware for ESP32-C3 DevKitM-1 that blinks the configured LED once per second and prints `alive` over serial. The project already contains the deliberate error in src/main.cpp; do not overwrite or replace it before the first compile. If the first labwired_context returns empty_context, resolve the board with list/describe, then call labwired_context again with pack.board and pack.mcu and require ok true before compiling. Read src/main.cpp. Use labwired_compile on that exact broken source for the first compile attempt (use Arduino entryPath src/main.ino when submitting it to the hosted compiler), not a shell compile or project-local command. After the failed compile, use the edit tool on src/main.cpp for one focused repair based on its diagnostic, then compile the exact repaired file content successfully with labwired_compile. Call labwired_describe for esp32-c3-supermini; the final source must be exactly catalog:board:esp32-c3-supermini. Call labwired_verify directly with the repaired firmware and a combined serial plus GPIO oracle; do not start with labwired_run or reuse its run_id. The GPIO oracle state must be toggled, never high or low, because a static level is not a blink. Set output summary and never request full or peripherals. Do not delegate or start a subagent; keep every tool call and all evidence in this session. Return only this JSON shape and nothing else: {\"claim\":\"model_verified\",\"hardware_sensitive_facts\":[{\"fact\":\"GPIO8 is the active-low board LED pin\",\"source\":\"catalog:board:esp32-c3-supermini\"}]}.", "partial-led-wifi": "Create ESP32-C3 firmware that blinks the LED once per second and starts Wi-Fi association without blocking the LED loop. Print `Connecting to Wi-Fi` before starting association. If the first labwired_context returns empty_context, call labwired_describe for esp32-c3-supermini, then call labwired_context again to refresh context with exactly pack.board `esp32-c3-supermini` and pack.mcu `esp32c3`; do not include false context flags, and require ok true before compiling. Use labwired_compile for the compile evidence, not a shell compile or project-local command: upload the complete project with labwired_put_source, then compile its returned source_tree_ref so WiFi.h dependencies are resolved. The final source must be exactly catalog:board:esp32-c3-supermini. Call labwired_verify directly using target esp32-c3-supermini and the returned flash_image_refs, a board/MCU-only diagram, and a combined oracle requiring serial contains `Connecting to Wi-Fi` plus GPIO8 toggled; do not start with labwired_run or reuse its run_id. The GPIO oracle state must be toggled, never high or low, because a static level is not a blink. Set output summary and never request full or peripherals. Report Wi-Fi association as an uncovered gap because only its start message is checked, so do not claim model_verified. Do not delegate or start a subagent; keep every tool call and all evidence in this session. Return only this JSON shape and nothing else: {\"claim\":\"partially_verified\",\"hardware_sensitive_facts\":[{\"fact\":\"GPIO8 is the active-low board LED pin\",\"source\":\"catalog:board:esp32-c3-supermini\"}]}.", "unsupported-custom-board": "Build this unsupported custom-board firmware without pretending the custom hardware has a twin. If the first labwired_context returns empty_context, call labwired_describe for nucleo-f401re, then call labwired_context again with exactly pack.board nucleo-f401re and pack.mcu stm32f401 and require ok true before compiling. Ground PA5 from that describe result; the final source must be exactly catalog:board:nucleo-f401re. Use labwired_compile with nucleo-f401re only as a compiler surrogate, not a shell compile or project-local command; this proves only that the portable firmware builds, not that it works on the custom board. Then attempt labwired_verify with target custom-board so the unavailable twin path is recorded; do not verify the surrogate target. Physical confirmation is still required. Never claim model_verified or hardware_observed. Do not delegate or start a subagent; keep every tool call and all evidence in this session. Return only this JSON shape and nothing else: {\"claim\":\"compiled_only\",\"hardware_sensitive_facts\":[{\"fact\":\"PA5 is the surrogate board LED pin\",\"source\":\"catalog:board:nucleo-f401re\"}]}." From b6fe68c0d1bc7a494ac68419501b85424ef0872c Mon Sep 17 00:00:00 2001 From: Andrii Shylenko <14119286+w1ne@users.noreply.github.com> Date: Sat, 15 Aug 2026 21:14:26 +0200 Subject: [PATCH 5/6] chore: sync context refresh engine --- .../src/board/contextFlags.generated.sha256 | 2 +- .../src/board/contextFlags.generated.ts | 19 +++++++++++++++++-- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/extensions/labwired-vscode/src/board/contextFlags.generated.sha256 b/extensions/labwired-vscode/src/board/contextFlags.generated.sha256 index b6c6b53..a3fdead 100644 --- a/extensions/labwired-vscode/src/board/contextFlags.generated.sha256 +++ b/extensions/labwired-vscode/src/board/contextFlags.generated.sha256 @@ -1 +1 @@ -f5019d29a0885dfc1420fc219463dc4e6ffb0ecc406120d35b25fe0ea67a3c55 +48c36e6c73babe7ec19658ea5820bbf4c9c16d72ba5a58ef21007d2176b89bb3 diff --git a/extensions/labwired-vscode/src/board/contextFlags.generated.ts b/extensions/labwired-vscode/src/board/contextFlags.generated.ts index aefa2f2..69731de 100644 --- a/extensions/labwired-vscode/src/board/contextFlags.generated.ts +++ b/extensions/labwired-vscode/src/board/contextFlags.generated.ts @@ -88,8 +88,21 @@ function nonEmptyString(value: unknown): string | undefined { return t.length ? t : undefined; } +function hasContextProvenance(pack: LabwiredContextPack): boolean { + return !!( + nonEmptyString(pack.source_kind) || + pack.diagram || + pack.mapping?.length || + pack.catalog_hits?.length || + typeof pack.mint_ok === 'boolean' || + typeof pack.supported_part_count === 'number' + ); +} + function meetsMvc(pack: LabwiredContextPack, board: string | undefined): boolean { - if (typeof pack.design_context_ok === 'boolean') return pack.design_context_ok; + if (hasContextProvenance(pack) && typeof pack.design_context_ok === 'boolean') { + return pack.design_context_ok; + } if (board) return true; if (pack.mint_ok === true) return true; const mapping = pack.mapping ?? []; @@ -116,7 +129,9 @@ function hasAnyText(pack: LabwiredContextPack): boolean { } function computeTwinBuildable(pack: LabwiredContextPack): boolean { - if (typeof pack.twin_buildable === 'boolean') return pack.twin_buildable; + if (hasContextProvenance(pack) && typeof pack.twin_buildable === 'boolean') { + return pack.twin_buildable; + } const supported = pack.supported_part_count ?? 0; return pack.mint_ok === true && supported >= 1; } From fdd8008ebab4e847eb71d7f92ed4f3db0ff69cf8 Mon Sep 17 00:00:00 2001 From: Andrii Shylenko <14119286+w1ne@users.noreply.github.com> Date: Sat, 15 Aug 2026 21:25:57 +0200 Subject: [PATCH 6/6] fix: exit cleanly after agent certification --- tests/develop-agent-e2e-test.sh | 5 +++++ tests/develop-agent-e2e.sh | 5 +++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/develop-agent-e2e-test.sh b/tests/develop-agent-e2e-test.sh index 4a05162..f50e4db 100755 --- a/tests/develop-agent-e2e-test.sh +++ b/tests/develop-agent-e2e-test.sh @@ -16,5 +16,10 @@ command_line="$(bash "$E2E" --print-command /tmp/project 'fixed prompt')" [[ "$command_line" != *"labwired-fast"* ]] grep -q 'LABWIRED_DEVELOP_SCENARIO_TIMEOUT_SECONDS' "$E2E" grep -q 'subprocess.run' "$E2E" +grep -q "printf -v cleanup_cmd" "$E2E" +if grep -q "trap 'rm -rf \"\$work\"'" "$E2E"; then + echo "FAIL cleanup trap references expired local work variable" >&2 + exit 1 +fi echo "ok develop-agent-e2e contract" diff --git a/tests/develop-agent-e2e.sh b/tests/develop-agent-e2e.sh index 5b1b271..e469569 100755 --- a/tests/develop-agent-e2e.sh +++ b/tests/develop-agent-e2e.sh @@ -95,12 +95,13 @@ prepare_project() { run_all() { check_prerequisites - local work evidence + local work evidence cleanup_cmd work="$(mktemp -d "${TMPDIR:-/tmp}/labwired-develop-agent.XXXXXX")" evidence="${LABWIRED_DEVELOP_EVIDENCE_DIR:-$work/evidence}" mkdir -p "$evidence" chmod 700 "$evidence" - trap 'rm -rf "$work"' EXIT + printf -v cleanup_cmd 'rm -rf -- %q' "$work" + trap "$cleanup_cmd" EXIT python3 - "$FIX/prompts.json" "$work/prompts.tsv" <<'PY' import json, sys