From 9895dfdf75bcac980e456473a9bd4cdce38c6a93 Mon Sep 17 00:00:00 2001 From: Tyler Kendrick <145080887+Tyler-R-Kendrick@users.noreply.github.com> Date: Sun, 26 Jul 2026 14:38:58 -0500 Subject: [PATCH] feat(tree-edit): add pack-owned set-property action --- .../var1-02-set-property-action-20260726.json | 559 ++++++++++++++++++ .../var1-02-set-property-action-20260726.md | 131 ++++ src/slm_training/dsl/variants.py | 2 +- .../experiments/slm299_edit_reachability.py | 62 +- src/slm_training/models/checkpoint_migrate.py | 12 +- .../models/tree_edit_diffusion.py | 162 ++++- .../resources/variant_registry.json | 8 +- src/slm_training/resources/versions.json | 25 +- .../test_slm299_edit_reachability.py | 17 + tests/test_models/test_tree_edit_diffusion.py | 99 +++- 10 files changed, 1029 insertions(+), 48 deletions(-) create mode 100644 docs/design/var1-02-set-property-action-20260726.json create mode 100644 docs/design/var1-02-set-property-action-20260726.md diff --git a/docs/design/var1-02-set-property-action-20260726.json b/docs/design/var1-02-set-property-action-20260726.json new file mode 100644 index 000000000..1e5675766 --- /dev/null +++ b/docs/design/var1-02-set-property-action-20260726.json @@ -0,0 +1,559 @@ +{ + "schema": "slm299_edit_reachability_audit/v2", + "experiment_id": "slm299-edit-reachability", + "seed_source": "root = Stack([], \"column\")", + "mode": "extended", + "max_edits": 8, + "node_budget": 15, + "generated_at": "2026-07-26T19:32:41Z", + "claim_class": "capability", + "production_action_space_changed": true, + "slm425_set_property": { + "action": "ACTION_SET_PROPERTY", + "property_authority": "DslPack.component_property_domains", + "production_domain": "the currently declared per-component rest values only; no hypothetical row widening", + "checkpoint_format": 3, + "comparison_scope": "full current smoke/held_out/adversarial/ood/rico corpus at max_edits=8,node_budget=15; train corpus unavailable", + "slm305_baseline": "docs/design/iter-slm305-edit-language-20260724.json", + "var1_01_probe_arm_b": "docs/design/var1-01-set-property-probe-20260725.json", + "per_suite_delta": { + "smoke": "0/3 reachable unchanged from SLM-305 and VAR1-01 arm B", + "held_out": "0/5 reachable unchanged from SLM-305 and VAR1-01 arm B", + "adversarial": "one flip: adv_empty_prompt_01 PROVEN_UNREACHABLE(needs_direction_change) -> PROVEN_REACHABLE in 2 edits via ADD then SET_PROPERTY; 1/3 decided (0.333333), at or below arm B's 1/2 decided (0.5)", + "ood": "0/4 reachable unchanged from SLM-305 and VAR1-01 arm B", + "rico": "0/1 decided reachable and 34 UNKNOWN_BUDGET; no favorable comparison to the 6-case SLM-305 historical snapshot or arm B's 35 UNKNOWN_BUDGET cases", + "train": "current frozen train corpus unavailable; no comparison claimed" + }, + "honest_caveat": "This is bounded local reachability evidence only. It makes no quality, ship, promotion, checkpoint, or meaningful-parse claim. UNKNOWN_BUDGET is not unreachability evidence." + }, + "verdict_policy": "reachable_fraction is computed over decided cases only; UNKNOWN_BUDGET cases are reported separately and are never counted as unreachable; suites without a corpus are corpus_unavailable, never zero-reachable. Reachability is a space-coverage proof, never a model-quality claim.", + "suites": { + "train": { + "status": "corpus_unavailable", + "n_cases": 0, + "reachable_fraction": null + }, + "smoke": { + "status": "ok", + "n_cases": 3, + "n_decided": 3, + "n_unknown_budget": 0, + "n_reachable": 0, + "reachable_fraction": 0.0, + "reason_histogram": { + "unsupported_component": 3 + }, + "action_coverage": {}, + "component_coverage": {}, + "edit_lower_bound": null, + "cases": [ + { + "id": "smoke_hero_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "unsupported_component", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "smoke_button_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "unsupported_component", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "smoke_callout_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "unsupported_component", + "edit_lower_bound": null, + "path_length": 0 + } + ] + }, + "held_out": { + "status": "ok", + "n_cases": 5, + "n_decided": 5, + "n_unknown_budget": 0, + "n_reachable": 0, + "reachable_fraction": 0.0, + "reason_histogram": { + "unsupported_component": 5 + }, + "action_coverage": {}, + "component_coverage": {}, + "edit_lower_bound": null, + "cases": [ + { + "id": "held_out_form_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "unsupported_component", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "held_out_dual_card_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "unsupported_component", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "held_out_input_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "unsupported_component", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "held_out_tabs_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "unsupported_component", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "held_out_settings_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "unsupported_component", + "edit_lower_bound": null, + "path_length": 0 + } + ] + }, + "adversarial": { + "status": "ok", + "n_cases": 4, + "n_decided": 3, + "n_unknown_budget": 1, + "n_reachable": 1, + "reachable_fraction": 0.333333, + "reason_histogram": { + "budget": 1, + "needs_direction_change": 1, + "reached": 1, + "unsupported_component": 1 + }, + "action_coverage": { + "ADD": 1, + "SET_PROPERTY": 1 + }, + "component_coverage": { + "TextContent": 1 + }, + "edit_lower_bound": { + "min": 2, + "median": 2, + "max": 2, + "mean": 2 + }, + "cases": [ + { + "id": "adv_empty_prompt_01", + "verdict": "PROVEN_REACHABLE", + "reason_code": "reached", + "edit_lower_bound": 2, + "path_length": 2 + }, + { + "id": "adv_dual_card_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "unsupported_component", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "adv_deep_nest_01", + "verdict": "UNKNOWN_BUDGET", + "reason_code": "budget", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "adv_many_buttons_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "needs_direction_change", + "edit_lower_bound": null, + "path_length": 0 + } + ] + }, + "ood": { + "status": "ok", + "n_cases": 4, + "n_decided": 4, + "n_unknown_budget": 0, + "n_reachable": 0, + "reachable_fraction": 0.0, + "reason_histogram": { + "unsupported_component": 4 + }, + "action_coverage": {}, + "component_coverage": {}, + "edit_lower_bound": null, + "cases": [ + { + "id": "ood_dashboard_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "unsupported_component", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "ood_gallery_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "unsupported_component", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "ood_modal_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "unsupported_component", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "ood_auth_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "unsupported_component", + "edit_lower_bound": null, + "path_length": 0 + } + ] + }, + "rico": { + "status": "ok", + "n_cases": 35, + "n_decided": 1, + "n_unknown_budget": 34, + "n_reachable": 0, + "reachable_fraction": 0.0, + "reason_histogram": { + "budget": 34, + "needs_direction_change": 1 + }, + "action_coverage": {}, + "component_coverage": {}, + "edit_lower_bound": null, + "cases": [ + { + "id": "rico_eval_test_0", + "verdict": "UNKNOWN_BUDGET", + "reason_code": "budget", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "rico_eval_test_1", + "verdict": "UNKNOWN_BUDGET", + "reason_code": "budget", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "rico_eval_test_2", + "verdict": "UNKNOWN_BUDGET", + "reason_code": "budget", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "rico_eval_test_4", + "verdict": "UNKNOWN_BUDGET", + "reason_code": "budget", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "rico_eval_test_8", + "verdict": "UNKNOWN_BUDGET", + "reason_code": "budget", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "rico_eval_test_9", + "verdict": "UNKNOWN_BUDGET", + "reason_code": "budget", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "rico_eval_test_12", + "verdict": "UNKNOWN_BUDGET", + "reason_code": "budget", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "rico_eval_test_17", + "verdict": "UNKNOWN_BUDGET", + "reason_code": "budget", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "rico_eval_test_20", + "verdict": "UNKNOWN_BUDGET", + "reason_code": "budget", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "rico_eval_test_25", + "verdict": "UNKNOWN_BUDGET", + "reason_code": "budget", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "rico_eval_test_34", + "verdict": "UNKNOWN_BUDGET", + "reason_code": "budget", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "rico_eval_test_35", + "verdict": "UNKNOWN_BUDGET", + "reason_code": "budget", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "rico_eval_test_38", + "verdict": "UNKNOWN_BUDGET", + "reason_code": "budget", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "rico_eval_test_40", + "verdict": "UNKNOWN_BUDGET", + "reason_code": "budget", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "rico_eval_test_41", + "verdict": "UNKNOWN_BUDGET", + "reason_code": "budget", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "rico_eval_test_42", + "verdict": "UNKNOWN_BUDGET", + "reason_code": "budget", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "rico_eval_test_47", + "verdict": "UNKNOWN_BUDGET", + "reason_code": "budget", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "rico_eval_test_48", + "verdict": "UNKNOWN_BUDGET", + "reason_code": "budget", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "rico_eval_test_51", + "verdict": "UNKNOWN_BUDGET", + "reason_code": "budget", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "rico_eval_test_53", + "verdict": "UNKNOWN_BUDGET", + "reason_code": "budget", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "rico_eval_test_55", + "verdict": "UNKNOWN_BUDGET", + "reason_code": "budget", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "rico_eval_test_56", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "needs_direction_change", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "rico_eval_test_57", + "verdict": "UNKNOWN_BUDGET", + "reason_code": "budget", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "rico_eval_test_58", + "verdict": "UNKNOWN_BUDGET", + "reason_code": "budget", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "rico_eval_test_59", + "verdict": "UNKNOWN_BUDGET", + "reason_code": "budget", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "rico_eval_test_60", + "verdict": "UNKNOWN_BUDGET", + "reason_code": "budget", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "rico_eval_test_68", + "verdict": "UNKNOWN_BUDGET", + "reason_code": "budget", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "rico_eval_test_69", + "verdict": "UNKNOWN_BUDGET", + "reason_code": "budget", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "rico_eval_test_77", + "verdict": "UNKNOWN_BUDGET", + "reason_code": "budget", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "rico_eval_test_81", + "verdict": "UNKNOWN_BUDGET", + "reason_code": "budget", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "rico_eval_test_91", + "verdict": "UNKNOWN_BUDGET", + "reason_code": "budget", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "rico_eval_test_95", + "verdict": "UNKNOWN_BUDGET", + "reason_code": "budget", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "rico_eval_test_97", + "verdict": "UNKNOWN_BUDGET", + "reason_code": "budget", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "rico_eval_test_99", + "verdict": "UNKNOWN_BUDGET", + "reason_code": "budget", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "rico_eval_test_104", + "verdict": "UNKNOWN_BUDGET", + "reason_code": "budget", + "edit_lower_bound": null, + "path_length": 0 + } + ] + } + }, + "version_stamp": { + "stamp_schema": "version_stamp/v1", + "code_commit": "74e6cca3f84cae21bf2a4b11055f1c9495f144c9", + "code_dirty": true, + "components": { + "harness.experiments.slm299_edit_reachability": "v6" + }, + "stamped_at": "2026-07-26T19:34:30.301938+00:00" + }, + "x22_evidence_annotations": [ + { + "date": "2026-07-26T19:32:41Z", + "target_doc": "docs/design/iter-x22-d3-kapur-tree-edit-20260717.md", + "suite": "adversarial", + "annotation": "SLM-299 edit-reachability audit: from the standard X22 minimal seed, reachable_fraction=0.333333 over 3 decided adversarial cases (1 UNKNOWN_BUDGET, never counted unreachable). Suite-level quality readings of the X22 tree-edit decode on adversarial in this document are bounded by that fraction: the unreachable share of gold programs cannot be produced by the decode space at all, so measured quality on those cases reflects space coverage, not model quality." + }, + { + "date": "2026-07-26T19:32:41Z", + "target_doc": "docs/design/iter-efs0-04-x22-reproduction-20260717.md", + "suite": "adversarial", + "annotation": "SLM-299 edit-reachability audit: from the standard X22 minimal seed, reachable_fraction=0.333333 over 3 decided adversarial cases (1 UNKNOWN_BUDGET, never counted unreachable). Suite-level quality readings of the X22 tree-edit decode on adversarial in this document are bounded by that fraction: the unreachable share of gold programs cannot be produced by the decode space at all, so measured quality on those cases reflects space coverage, not model quality." + }, + { + "date": "2026-07-26T19:32:41Z", + "target_doc": "docs/design/iter-x22-d3-kapur-tree-edit-20260717.md", + "suite": "held_out", + "annotation": "SLM-299 edit-reachability audit: from the standard X22 minimal seed, reachable_fraction=0.0 over 5 decided held_out cases (0 UNKNOWN_BUDGET, never counted unreachable). Suite-level quality readings of the X22 tree-edit decode on held_out in this document are bounded by that fraction: the unreachable share of gold programs cannot be produced by the decode space at all, so measured quality on those cases reflects space coverage, not model quality." + }, + { + "date": "2026-07-26T19:32:41Z", + "target_doc": "docs/design/iter-efs0-04-x22-reproduction-20260717.md", + "suite": "held_out", + "annotation": "SLM-299 edit-reachability audit: from the standard X22 minimal seed, reachable_fraction=0.0 over 5 decided held_out cases (0 UNKNOWN_BUDGET, never counted unreachable). Suite-level quality readings of the X22 tree-edit decode on held_out in this document are bounded by that fraction: the unreachable share of gold programs cannot be produced by the decode space at all, so measured quality on those cases reflects space coverage, not model quality." + }, + { + "date": "2026-07-26T19:32:41Z", + "target_doc": "docs/design/iter-x22-d3-kapur-tree-edit-20260717.md", + "suite": "ood", + "annotation": "SLM-299 edit-reachability audit: from the standard X22 minimal seed, reachable_fraction=0.0 over 4 decided ood cases (0 UNKNOWN_BUDGET, never counted unreachable). Suite-level quality readings of the X22 tree-edit decode on ood in this document are bounded by that fraction: the unreachable share of gold programs cannot be produced by the decode space at all, so measured quality on those cases reflects space coverage, not model quality." + }, + { + "date": "2026-07-26T19:32:41Z", + "target_doc": "docs/design/iter-efs0-04-x22-reproduction-20260717.md", + "suite": "ood", + "annotation": "SLM-299 edit-reachability audit: from the standard X22 minimal seed, reachable_fraction=0.0 over 4 decided ood cases (0 UNKNOWN_BUDGET, never counted unreachable). Suite-level quality readings of the X22 tree-edit decode on ood in this document are bounded by that fraction: the unreachable share of gold programs cannot be produced by the decode space at all, so measured quality on those cases reflects space coverage, not model quality." + }, + { + "date": "2026-07-26T19:32:41Z", + "target_doc": "docs/design/iter-x22-d3-kapur-tree-edit-20260717.md", + "suite": "rico", + "annotation": "SLM-299 edit-reachability audit: from the standard X22 minimal seed, reachable_fraction=0.0 over 1 decided rico cases (34 UNKNOWN_BUDGET, never counted unreachable). Suite-level quality readings of the X22 tree-edit decode on rico in this document are bounded by that fraction: the unreachable share of gold programs cannot be produced by the decode space at all, so measured quality on those cases reflects space coverage, not model quality." + }, + { + "date": "2026-07-26T19:32:41Z", + "target_doc": "docs/design/iter-efs0-04-x22-reproduction-20260717.md", + "suite": "rico", + "annotation": "SLM-299 edit-reachability audit: from the standard X22 minimal seed, reachable_fraction=0.0 over 1 decided rico cases (34 UNKNOWN_BUDGET, never counted unreachable). Suite-level quality readings of the X22 tree-edit decode on rico in this document are bounded by that fraction: the unreachable share of gold programs cannot be produced by the decode space at all, so measured quality on those cases reflects space coverage, not model quality." + }, + { + "date": "2026-07-26T19:32:41Z", + "target_doc": "docs/design/iter-x22-d3-kapur-tree-edit-20260717.md", + "suite": "smoke", + "annotation": "SLM-299 edit-reachability audit: from the standard X22 minimal seed, reachable_fraction=0.0 over 3 decided smoke cases (0 UNKNOWN_BUDGET, never counted unreachable). Suite-level quality readings of the X22 tree-edit decode on smoke in this document are bounded by that fraction: the unreachable share of gold programs cannot be produced by the decode space at all, so measured quality on those cases reflects space coverage, not model quality." + }, + { + "date": "2026-07-26T19:32:41Z", + "target_doc": "docs/design/iter-efs0-04-x22-reproduction-20260717.md", + "suite": "smoke", + "annotation": "SLM-299 edit-reachability audit: from the standard X22 minimal seed, reachable_fraction=0.0 over 3 decided smoke cases (0 UNKNOWN_BUDGET, never counted unreachable). Suite-level quality readings of the X22 tree-edit decode on smoke in this document are bounded by that fraction: the unreachable share of gold programs cannot be produced by the decode space at all, so measured quality on those cases reflects space coverage, not model quality." + } + ] +} diff --git a/docs/design/var1-02-set-property-action-20260726.md b/docs/design/var1-02-set-property-action-20260726.md new file mode 100644 index 000000000..cc656e819 --- /dev/null +++ b/docs/design/var1-02-set-property-action-20260726.md @@ -0,0 +1,131 @@ +# SLM-299 (LAR1-03): X22 edit-space reachability audit + +- generated_at: `2026-07-26T19:32:41Z` +- seed: `root = Stack([], "column")` +- mode: `extended` +- max_edits: 8, node_budget: 15 +- verdict policy: reachable_fraction is computed over decided cases only; UNKNOWN_BUDGET cases are reported separately and are never counted as unreachable; suites without a corpus are corpus_unavailable, never zero-reachable. Reachability is a space-coverage proof, never a model-quality claim. + +> Reachability is space coverage, not model quality: no quality claim +> follows from these proofs alone. + +## SLM-425 production delta + +`ACTION_SET_PROPERTY` is now a real format-3 action. It mutates only the +target container's finite, pack-owned `rest` domain and fragment-validates the +structured rebuilt statement. The broader VAR1-01 `row` control remains +hypothetical; it is not production authority. + +| suite | SLM-305 baseline | VAR1-01 arm B | SLM-425 production | delta / caveat | +| --- | --- | --- | --- | --- | +| train | 0/6 | unavailable | unavailable | Current frozen train corpus is unavailable; no comparison claimed. | +| smoke | 0/3 | 0/3 | 0/3 | No verdict flips. | +| held_out | 0/5 | 0/5 | 0/5 | No verdict flips. | +| adversarial | 0/4 | 1/2 decided (0.5) | 1/3 decided (0.333333) | `adv_empty_prompt_01` flips `needs_direction_change` → reachable in 2 edits (`ADD`, `SET_PROPERTY`); production is at or below arm B. | +| ood | 0/4 | 0/4 | 0/4 | No verdict flips. | +| rico | 0/6 historical | 35 unknown | 0/1 decided; 34 unknown | No favorable comparison: historical corpus size differs and UNKNOWN_BUDGET is not unreachability evidence. | + +This bounded local run used `max_edits=8`, `node_budget=15`, and the full +currently available smoke/held_out/adversarial/ood/rico inputs. It is a +capability-space coverage result only: no quality, ship, promotion, checkpoint, +or meaningful-parse claim follows from it. + +## Suite summary + +| suite | n | decided | unknown | reachable_fraction | min/med/max edits | +| --- | --- | --- | --- | --- | --- | +| train | corpus_unavailable | — | — | — | — | +| smoke | 3 | 3 | 0 | 0.0 | — | +| held_out | 5 | 5 | 0 | 0.0 | — | +| adversarial | 4 | 3 | 1 | 0.333333 | 2/2/2 | +| ood | 4 | 4 | 0 | 0.0 | — | +| rico | 35 | 1 | 34 | 0.0 | — | + +## Reason-code histograms + +- **smoke**: `{"unsupported_component": 3}` +- **held_out**: `{"unsupported_component": 5}` +- **adversarial**: `{"budget": 1, "needs_direction_change": 1, "reached": 1, "unsupported_component": 1}` +- **ood**: `{"unsupported_component": 4}` +- **rico**: `{"budget": 34, "needs_direction_change": 1}` + +## Action / component coverage (reachable paths) + +- **smoke**: actions `{}` components `{}` +- **held_out**: actions `{}` components `{}` +- **adversarial**: actions `{"ADD": 1, "SET_PROPERTY": 1}` components `{"TextContent": 1}` +- **ood**: actions `{}` components `{}` +- **rico**: actions `{}` components `{}` + +## Per-case verdicts + +- **train**: corpus unavailable +- **smoke**: + - `smoke_hero_01` PROVEN_UNREACHABLE (unsupported_component, lower_bound=None) + - `smoke_button_01` PROVEN_UNREACHABLE (unsupported_component, lower_bound=None) + - `smoke_callout_01` PROVEN_UNREACHABLE (unsupported_component, lower_bound=None) +- **held_out**: + - `held_out_form_01` PROVEN_UNREACHABLE (unsupported_component, lower_bound=None) + - `held_out_dual_card_01` PROVEN_UNREACHABLE (unsupported_component, lower_bound=None) + - `held_out_input_01` PROVEN_UNREACHABLE (unsupported_component, lower_bound=None) + - `held_out_tabs_01` PROVEN_UNREACHABLE (unsupported_component, lower_bound=None) + - `held_out_settings_01` PROVEN_UNREACHABLE (unsupported_component, lower_bound=None) +- **adversarial**: + - `adv_empty_prompt_01` PROVEN_REACHABLE (reached, lower_bound=2) + - `adv_dual_card_01` PROVEN_UNREACHABLE (unsupported_component, lower_bound=None) + - `adv_deep_nest_01` UNKNOWN_BUDGET (budget, lower_bound=None) + - `adv_many_buttons_01` PROVEN_UNREACHABLE (needs_direction_change, lower_bound=None) +- **ood**: + - `ood_dashboard_01` PROVEN_UNREACHABLE (unsupported_component, lower_bound=None) + - `ood_gallery_01` PROVEN_UNREACHABLE (unsupported_component, lower_bound=None) + - `ood_modal_01` PROVEN_UNREACHABLE (unsupported_component, lower_bound=None) + - `ood_auth_01` PROVEN_UNREACHABLE (unsupported_component, lower_bound=None) +- **rico**: + - `rico_eval_test_0` UNKNOWN_BUDGET (budget, lower_bound=None) + - `rico_eval_test_1` UNKNOWN_BUDGET (budget, lower_bound=None) + - `rico_eval_test_2` UNKNOWN_BUDGET (budget, lower_bound=None) + - `rico_eval_test_4` UNKNOWN_BUDGET (budget, lower_bound=None) + - `rico_eval_test_8` UNKNOWN_BUDGET (budget, lower_bound=None) + - `rico_eval_test_9` UNKNOWN_BUDGET (budget, lower_bound=None) + - `rico_eval_test_12` UNKNOWN_BUDGET (budget, lower_bound=None) + - `rico_eval_test_17` UNKNOWN_BUDGET (budget, lower_bound=None) + - `rico_eval_test_20` UNKNOWN_BUDGET (budget, lower_bound=None) + - `rico_eval_test_25` UNKNOWN_BUDGET (budget, lower_bound=None) + - `rico_eval_test_34` UNKNOWN_BUDGET (budget, lower_bound=None) + - `rico_eval_test_35` UNKNOWN_BUDGET (budget, lower_bound=None) + - `rico_eval_test_38` UNKNOWN_BUDGET (budget, lower_bound=None) + - `rico_eval_test_40` UNKNOWN_BUDGET (budget, lower_bound=None) + - `rico_eval_test_41` UNKNOWN_BUDGET (budget, lower_bound=None) + - `rico_eval_test_42` UNKNOWN_BUDGET (budget, lower_bound=None) + - `rico_eval_test_47` UNKNOWN_BUDGET (budget, lower_bound=None) + - `rico_eval_test_48` UNKNOWN_BUDGET (budget, lower_bound=None) + - `rico_eval_test_51` UNKNOWN_BUDGET (budget, lower_bound=None) + - `rico_eval_test_53` UNKNOWN_BUDGET (budget, lower_bound=None) + - `rico_eval_test_55` UNKNOWN_BUDGET (budget, lower_bound=None) + - `rico_eval_test_56` PROVEN_UNREACHABLE (needs_direction_change, lower_bound=None) + - `rico_eval_test_57` UNKNOWN_BUDGET (budget, lower_bound=None) + - `rico_eval_test_58` UNKNOWN_BUDGET (budget, lower_bound=None) + - `rico_eval_test_59` UNKNOWN_BUDGET (budget, lower_bound=None) + - `rico_eval_test_60` UNKNOWN_BUDGET (budget, lower_bound=None) + - `rico_eval_test_68` UNKNOWN_BUDGET (budget, lower_bound=None) + - `rico_eval_test_69` UNKNOWN_BUDGET (budget, lower_bound=None) + - `rico_eval_test_77` UNKNOWN_BUDGET (budget, lower_bound=None) + - `rico_eval_test_81` UNKNOWN_BUDGET (budget, lower_bound=None) + - `rico_eval_test_91` UNKNOWN_BUDGET (budget, lower_bound=None) + - `rico_eval_test_95` UNKNOWN_BUDGET (budget, lower_bound=None) + - `rico_eval_test_97` UNKNOWN_BUDGET (budget, lower_bound=None) + - `rico_eval_test_99` UNKNOWN_BUDGET (budget, lower_bound=None) + - `rico_eval_test_104` UNKNOWN_BUDGET (budget, lower_bound=None) + +## X22 evidence annotations (append-only) + +- `docs/design/iter-x22-d3-kapur-tree-edit-20260717.md` [adversarial]: SLM-299 edit-reachability audit: from the standard X22 minimal seed, reachable_fraction=0.333333 over 3 decided adversarial cases (1 UNKNOWN_BUDGET, never counted unreachable). Suite-level quality readings of the X22 tree-edit decode on adversarial in this document are bounded by that fraction: the unreachable share of gold programs cannot be produced by the decode space at all, so measured quality on those cases reflects space coverage, not model quality. +- `docs/design/iter-efs0-04-x22-reproduction-20260717.md` [adversarial]: SLM-299 edit-reachability audit: from the standard X22 minimal seed, reachable_fraction=0.333333 over 3 decided adversarial cases (1 UNKNOWN_BUDGET, never counted unreachable). Suite-level quality readings of the X22 tree-edit decode on adversarial in this document are bounded by that fraction: the unreachable share of gold programs cannot be produced by the decode space at all, so measured quality on those cases reflects space coverage, not model quality. +- `docs/design/iter-x22-d3-kapur-tree-edit-20260717.md` [held_out]: SLM-299 edit-reachability audit: from the standard X22 minimal seed, reachable_fraction=0.0 over 5 decided held_out cases (0 UNKNOWN_BUDGET, never counted unreachable). Suite-level quality readings of the X22 tree-edit decode on held_out in this document are bounded by that fraction: the unreachable share of gold programs cannot be produced by the decode space at all, so measured quality on those cases reflects space coverage, not model quality. +- `docs/design/iter-efs0-04-x22-reproduction-20260717.md` [held_out]: SLM-299 edit-reachability audit: from the standard X22 minimal seed, reachable_fraction=0.0 over 5 decided held_out cases (0 UNKNOWN_BUDGET, never counted unreachable). Suite-level quality readings of the X22 tree-edit decode on held_out in this document are bounded by that fraction: the unreachable share of gold programs cannot be produced by the decode space at all, so measured quality on those cases reflects space coverage, not model quality. +- `docs/design/iter-x22-d3-kapur-tree-edit-20260717.md` [ood]: SLM-299 edit-reachability audit: from the standard X22 minimal seed, reachable_fraction=0.0 over 4 decided ood cases (0 UNKNOWN_BUDGET, never counted unreachable). Suite-level quality readings of the X22 tree-edit decode on ood in this document are bounded by that fraction: the unreachable share of gold programs cannot be produced by the decode space at all, so measured quality on those cases reflects space coverage, not model quality. +- `docs/design/iter-efs0-04-x22-reproduction-20260717.md` [ood]: SLM-299 edit-reachability audit: from the standard X22 minimal seed, reachable_fraction=0.0 over 4 decided ood cases (0 UNKNOWN_BUDGET, never counted unreachable). Suite-level quality readings of the X22 tree-edit decode on ood in this document are bounded by that fraction: the unreachable share of gold programs cannot be produced by the decode space at all, so measured quality on those cases reflects space coverage, not model quality. +- `docs/design/iter-x22-d3-kapur-tree-edit-20260717.md` [rico]: SLM-299 edit-reachability audit: from the standard X22 minimal seed, reachable_fraction=0.0 over 1 decided rico cases (34 UNKNOWN_BUDGET, never counted unreachable). Suite-level quality readings of the X22 tree-edit decode on rico in this document are bounded by that fraction: the unreachable share of gold programs cannot be produced by the decode space at all, so measured quality on those cases reflects space coverage, not model quality. +- `docs/design/iter-efs0-04-x22-reproduction-20260717.md` [rico]: SLM-299 edit-reachability audit: from the standard X22 minimal seed, reachable_fraction=0.0 over 1 decided rico cases (34 UNKNOWN_BUDGET, never counted unreachable). Suite-level quality readings of the X22 tree-edit decode on rico in this document are bounded by that fraction: the unreachable share of gold programs cannot be produced by the decode space at all, so measured quality on those cases reflects space coverage, not model quality. +- `docs/design/iter-x22-d3-kapur-tree-edit-20260717.md` [smoke]: SLM-299 edit-reachability audit: from the standard X22 minimal seed, reachable_fraction=0.0 over 3 decided smoke cases (0 UNKNOWN_BUDGET, never counted unreachable). Suite-level quality readings of the X22 tree-edit decode on smoke in this document are bounded by that fraction: the unreachable share of gold programs cannot be produced by the decode space at all, so measured quality on those cases reflects space coverage, not model quality. +- `docs/design/iter-efs0-04-x22-reproduction-20260717.md` [smoke]: SLM-299 edit-reachability audit: from the standard X22 minimal seed, reachable_fraction=0.0 over 3 decided smoke cases (0 UNKNOWN_BUDGET, never counted unreachable). Suite-level quality readings of the X22 tree-edit decode on smoke in this document are bounded by that fraction: the unreachable share of gold programs cannot be produced by the decode space at all, so measured quality on those cases reflects space coverage, not model quality. diff --git a/src/slm_training/dsl/variants.py b/src/slm_training/dsl/variants.py index 449f18fc2..b663d2215 100644 --- a/src/slm_training/dsl/variants.py +++ b/src/slm_training/dsl/variants.py @@ -247,7 +247,7 @@ def build_variant_contracts() -> tuple[VariantContractV1, ...]: pack_id="openui", action_alphabet_id="tree_edit_diffusion.edit_actions", action_alphabet_fingerprint=_alphabet_fingerprint(_tree_edit_action_names()), - kernel_ops=(), + kernel_ops=("openui.set_property",), seed_policy_id="tree_edit_diffusion.minimal_valid_program_seed", inventory_source="pack", source_path=_TREE_EDIT_DIFFUSION_SOURCE, diff --git a/src/slm_training/harnesses/experiments/slm299_edit_reachability.py b/src/slm_training/harnesses/experiments/slm299_edit_reachability.py index 91e646fa1..856d23c75 100644 --- a/src/slm_training/harnesses/experiments/slm299_edit_reachability.py +++ b/src/slm_training/harnesses/experiments/slm299_edit_reachability.py @@ -42,6 +42,7 @@ ACTION_REPLACE, ACTION_REPLACE_STATEMENT, ACTION_REPLACE_SUBTREE, + ACTION_SET_PROPERTY, MAX_SLOTS, Edit, Statement, @@ -380,13 +381,14 @@ def _check_invariants( v1 space: REPLACE preserves container-ness, arity, and the container's enum/direction arg; ADD creates leaves only and binds only inventory - slots; REMOVE deletes leaves only. Extended (SLM-305) space: ADD_CONTAINER + slots; REMOVE deletes leaves only. Extended (SLM-425) space: ADD_CONTAINER / INSERT_SUBTREE mint containers carrying one of ``CONTAINER_RESTS`` and bind only inventory slots, REPLACE_SUBTREE / BIND_PLACEHOLDER rebind leaves only to inventory slots, and the V0.5 statement actions mint only canonical - templates — the invariant reasons fire only when the corresponding REAL - action of the analyzed mode is absent. + templates. ``ACTION_SET_PROPERTY`` (SLM-425) mutates a declared + pack-owned container property, root included; target rests outside that + finite domain remain an exact impossibility proof. """ seed_containers = [s for s in seed if s.has_list] target_containers = [s for s in target if s.has_list] @@ -431,17 +433,27 @@ def _check_invariants( if normalized not in inventory: return REASON_NEEDS_SLOT_REBIND - # REPLACE preserves the container's raw enum/direction arg text (rest); - # no real action edits it. Containers minted by the container-creating - # actions carry exactly one of CONTAINER_RESTS. Every target container - # rest must therefore be a seed rest or (with container_add) a candidate - # mint rest; without any container-creating action the multisets must - # match exactly. + # The VAR1-01 synthetic what-if may alter any parser-legal rest. The real + # SLM-425 action is deliberately narrower: every target rest must appear + # in that target component's declared pack domain. if "set_property" in capabilities: # A rest-mutation action can produce any target rest on any # container, root included -- skip both rest checks below entirely. return None + if extended: + for stmt in target_containers: + rest_domains = [ + space.property_values(stmt.comp, property_index) + for property_index, property_name in enumerate( + space.property_names(stmt.comp) + ) + if property_name == "rest" + ] + if not any(stmt.rest in values for values in rest_domains): + return REASON_NEEDS_DIRECTION_CHANGE + return None + seed_rests = sorted(s.rest for s in seed_containers) target_rests = sorted(s.rest for s in target_containers) # Root can never be removed or re-minted, and REPLACE preserves rest: @@ -473,8 +485,9 @@ def _enumerate_children( """All one-edit successors under the REAL action set of ``mode``, applied through ``TreeEditSpace.apply`` so preconditions and parser re-validation are the deployed ones by construction. ``v1`` enumerates the original - REPLACE/ADD/REMOVE set; ``extended`` (SLM-305) adds the container, - subtree, V0.5-statement, and placeholder-binding actions. Deterministic: + REPLACE/ADD/REMOVE set; ``extended`` (SLM-425) adds the container, + subtree, V0.5-statement, placeholder-binding, and property actions. + Deterministic: same state and mode always yield the same enumeration order. ``visited`` (canonical-key set) is a pure search-efficiency hook: states @@ -539,6 +552,33 @@ def _replace_subtree_ok(stmt: Statement) -> bool: continue # SLM-305 extended real actions. if stmt.has_list: + for property_index, property_name in enumerate(space.property_names(stmt.comp)): + if property_name != "rest": + continue + for value_index, value in enumerate( + space.property_values(stmt.comp, property_index) + ): + if value == stmt.rest: + continue + edit = Edit( + ACTION_SET_PROPERTY, + stmt_idx, + comp=property_index, + target=value_index, + ) + nxt = space.apply(statements, edit, inventory, pre) + if nxt is not None: + children.append( + ( + nxt, + { + "action": "SET_PROPERTY", + "stmt": stmt_idx, + "property": property_name, + "rest": value, + }, + ) + ) for comp_idx in container_comp_idxs: for rest_idx in range(len(space.container_rests)): edit = Edit(ACTION_ADD_CONTAINER, stmt_idx, comp_idx, diff --git a/src/slm_training/models/checkpoint_migrate.py b/src/slm_training/models/checkpoint_migrate.py index 46e738705..758d7c880 100644 --- a/src/slm_training/models/checkpoint_migrate.py +++ b/src/slm_training/models/checkpoint_migrate.py @@ -301,13 +301,13 @@ def migrate_tree_edit_checkpoint( output_checkpoint: Path | str, device: str = "cpu", ) -> dict: - """Warm-start a tree-edit-diffusion format-1 checkpoint to format 2. + """Warm-start an older tree-edit checkpoint to the current format. - SLM-305 grew ``action_head`` from 4 to ``N_ACTIONS`` rows (extended edit - language). All same-shape tensors are copied verbatim; the old action-head - rows are preserved in the first rows of the new head and the new action - rows stay randomly initialized. A ``.migrate.json`` report is written next - to the output checkpoint. + SLM-425 grows ``action_head`` from format 2's 11 rows to ``N_ACTIONS`` and + introduces a property-value head. Same-shape tensors are copied verbatim; + the old action rows are copied into the prefix and newly introduced rows + and heads keep their initialized values. A ``.migrate.json`` report is + written next to the output checkpoint. """ from slm_training.models.tree_edit_diffusion import ( TreeEditDiffusionConfig, diff --git a/src/slm_training/models/tree_edit_diffusion.py b/src/slm_training/models/tree_edit_diffusion.py index b34ec2e5c..e15ae3fca 100644 --- a/src/slm_training/models/tree_edit_diffusion.py +++ b/src/slm_training/models/tree_edit_diffusion.py @@ -62,7 +62,8 @@ ACTION_INSERT_STATEMENT = 8 # insert a canonical V0.5 state/query/mutation statement ACTION_REPLACE_STATEMENT = 9 # swap one canonical V0.5 statement for another ACTION_BIND_PLACEHOLDER = 10 # (re)bind a leaf's slot to an inventory placeholder -N_ACTIONS = 11 +ACTION_SET_PROPERTY = 11 # replace one pack-owned property on an existing container +N_ACTIONS = 12 MAX_STMTS = 24 MAX_SLOTS = 16 @@ -74,6 +75,7 @@ class EditDomain: leaf_components: tuple[str, ...] container_components: tuple[str, ...] container_rests: tuple[str, ...] + component_property_domains: dict[str, dict[str, tuple[str, ...]]] statement_templates: tuple[tuple[str, str], ...] @@ -91,6 +93,13 @@ def edit_domain(pack_id: str = "openui") -> EditDomain: leaf_components=tuple(pack.leaf_components), container_components=tuple(pack.container_components), container_rests=tuple(dict.fromkeys(rests)), + component_property_domains={ + component: { + property_name: tuple(values) + for property_name, values in properties.items() + } + for component, properties in pack.component_property_domains.items() + }, statement_templates=tuple(pack.statement_templates), ) @@ -240,8 +249,9 @@ class Edit: SLM-305: ``target``/``payload`` are NEW DEFAULTED fields only, so old pickles and comparisons keep working. ``payload`` carries the leaf component index (INSERT_SUBTREE / REPLACE_SUBTREE) or the canonical V0.5 - template index (INSERT_STATEMENT / REPLACE_STATEMENT); ``target`` is - reserved for secondary statement addressing. + template index (INSERT_STATEMENT / REPLACE_STATEMENT). ``target`` indexes + a pack-owned secondary domain: a container rest for the container actions, + or the selected property value for ``ACTION_SET_PROPERTY``. """ action: int @@ -272,8 +282,20 @@ def __init__( self.leaf_components = self.domain.leaf_components self.container_components = self.domain.container_components self.container_rests = self.domain.container_rests + self.component_property_domains = self.domain.component_property_domains self.statement_templates = self.domain.statement_templates + def property_names(self, component: str) -> tuple[str, ...]: + """Finite pack-owned property names for ``component``.""" + return tuple(self.component_property_domains.get(component, {})) + + def property_values(self, component: str, property_index: int) -> tuple[str, ...]: + """Finite values for a component property, or an empty domain.""" + names = self.property_names(component) + if not 0 <= property_index < len(names): + return () + return self.component_property_domains[component][names[property_index]] + def fresh_name(self, statements: list[Statement]) -> str: taken = {s.name for s in statements} for i in range(len(statements) + 8): @@ -545,6 +567,36 @@ def apply( return None placeholder = self._placeholder(inventory, edit.slot) target.rest = json.dumps(placeholder, ensure_ascii=False) + elif edit.action == ACTION_SET_PROPERTY: + # Rebuild a single container statement from its pack-owned property + # domain. ``Statement`` owns this structural representation; do not + # splice text or make the pack's property spellings decode authority. + if not (0 <= edit.stmt < len(working)): + return None + target = working[edit.stmt] + if not target.has_list: + return None + names = self.property_names(target.comp) + if not (0 <= edit.comp < len(names)): + return None + property_name = names[edit.comp] + values = self.property_values(target.comp, edit.comp) + if not (0 <= edit.target < len(values)): + return None + value = values[edit.target] + # The current canonical Statement schema exposes ``rest`` as its + # only mutable container property. A future pack property needs a + # corresponding structured field before this action can admit it. + if property_name != "rest" or value == target.rest: + return None + candidate = Statement( + target.name, target.comp, list(target.children), value, True + ) + try: + validate_output(candidate.render(), kind="statement") + except Exception: # noqa: BLE001 + return None + working[edit.stmt] = candidate else: return None if pre_validate is not None and not pre_validate(working): @@ -575,6 +627,7 @@ def sample_mutation( ACTION_INSERT_STATEMENT, ACTION_REPLACE_STATEMENT, ACTION_BIND_PLACEHOLDER, + ACTION_SET_PROPERTY, ) ) if kind == ACTION_REPLACE: @@ -817,6 +870,40 @@ def sample_mutation( continue inverse = Edit(ACTION_BIND_PLACEHOLDER, idx, slot=old_slot) return mutated, inverse + if kind == ACTION_SET_PROPERTY: + # Mutation = replace an existing representable property; + # inverse = select its old pack-owned value again. Root is a + # normal candidate here because it cannot be re-minted. + candidates = [ + (i, s, property_index, values) + for i, s in enumerate(statements) + if s.has_list + for property_index, property_name in enumerate(self.property_names(s.comp)) + if property_name == "rest" + for values in (self.property_values(s.comp, property_index),) + if s.rest in values and len(values) > 1 + ] + if not candidates: + continue + idx, stmt, property_index, values = rng.choice(candidates) + old_value_index = values.index(stmt.rest) + choices = [value_index for value_index in range(len(values)) if value_index != old_value_index] + mutation = Edit( + ACTION_SET_PROPERTY, + idx, + comp=property_index, + target=rng.choice(choices), + ) + mutated = self.apply(statements, mutation, inventory) + if mutated is None: + continue + inverse = Edit( + ACTION_SET_PROPERTY, + idx, + comp=property_index, + target=old_value_index, + ) + return mutated, inverse # Mutation = remove a leaf; inverse = ADD it back. removable = [ i @@ -905,7 +992,11 @@ class TreeEditPolicy(nn.Module): heads factorize the bounded edit and a value head scores the state.""" def __init__( - self, vocab_size: int, cfg: TreeEditDiffusionConfig, n_components: int + self, + vocab_size: int, + cfg: TreeEditDiffusionConfig, + n_components: int, + n_property_values: int, ) -> None: super().__init__() self.embed = nn.Embedding(vocab_size, cfg.d_model) @@ -921,6 +1012,7 @@ def __init__( self.stmt_head = nn.Linear(cfg.d_model, MAX_STMTS) self.comp_head = nn.Linear(cfg.d_model, n_components) self.slot_head = nn.Linear(cfg.d_model, MAX_SLOTS) + self.property_value_head = nn.Linear(cfg.d_model, n_property_values) self.value_head = nn.Linear(cfg.d_model, 1) def forward( @@ -943,6 +1035,7 @@ def forward( "stmt": self.stmt_head(pooled), "comp": self.comp_head(pooled), "slot": self.slot_head(pooled), + "property_value": self.property_value_head(pooled), "value": torch.sigmoid(self.value_head(pooled)).squeeze(-1), } @@ -950,10 +1043,11 @@ def forward( class TreeEditDiffusionModel(nn.Module): """Prompt-conditioned Kapur-style edit policy + value search (X22).""" - # Format 2 (SLM-305): action_head grew to N_ACTIONS=11 with the extended - # edit language. Format-1 checkpoints fail closed here; warm-start them + # Format 3 (SLM-425): action_head grew to N_ACTIONS=12 for SET_PROPERTY, + # and the property-value head was introduced. Earlier checkpoints fail + # closed here; warm-start them # via ``checkpoint_migrate.migrate_tree_edit_checkpoint``. - CHECKPOINT_FORMAT = 2 + CHECKPOINT_FORMAT = 3 def __init__( self, @@ -980,7 +1074,14 @@ def __init__( local_files_only=self.config.local_files_only, ) self.policy = TreeEditPolicy( - tokenizer.vocab_size, self.config, len(self.space.components) + tokenizer.vocab_size, + self.config, + len(self.space.components), + max( + len(values) + for properties in self.space.component_property_domains.values() + for values in properties.values() + ), ) self._rng = random.Random(self.config.seed) self.last_training_metrics: dict[str, float] = {} @@ -1151,6 +1252,17 @@ def training_loss(self, batch: list[ExampleRecord]) -> torch.Tensor: slot_loss = F.cross_entropy(out["slot"][idx], slot_t) loss = loss + slot_loss losses["slot"] = float(slot_loss.detach().cpu()) + property_rows = [ + i for i, e in enumerate(targets) if e.action == ACTION_SET_PROPERTY + ] + if property_rows: + idx = torch.tensor(property_rows, device=device) + property_t = torch.tensor( + [targets[i].target for i in property_rows], device=device + ) + property_loss = F.cross_entropy(out["property_value"][idx], property_t) + loss = loss + property_loss + losses["property_value"] = float(property_loss.detach().cpu()) value_t = torch.tensor(values, device=device, dtype=out["value"].dtype) value_loss = F.mse_loss(out["value"], value_t) loss = loss + value_loss @@ -1180,12 +1292,19 @@ def _seed_state(self, inventory: list[str]) -> list[Statement] | None: return None def _enumerate_edits( - self, out: dict[str, torch.Tensor], row: int, n_stmts: int, n_slots: int + self, + out: dict[str, torch.Tensor], + row: int, + n_stmts: int, + n_slots: int, + *, + statements: list[Statement] | None = None, ) -> list[tuple[float, Edit]]: action_lp = F.log_softmax(out["action"][row], dim=-1) stmt_lp = F.log_softmax(out["stmt"][row][: max(n_stmts, 1)], dim=-1) comp_lp = F.log_softmax(out["comp"][row], dim=-1) slot_lp = F.log_softmax(out["slot"][row][: max(n_slots, 1)], dim=-1) + property_value_lp = F.log_softmax(out["property_value"][row], dim=-1) scored: list[tuple[float, Edit]] = [ (float(action_lp[ACTION_STOP]), Edit(ACTION_STOP)) ] @@ -1267,6 +1386,29 @@ def _enumerate_edits( Edit(ACTION_REPLACE_STATEMENT, stmt, payload=payload), ) ) + statement = statements[stmt] if statements is not None else None + if statement is not None and statement.has_list: + for property_index, property_name in enumerate(self.space.property_names(statement.comp)): + if property_name != "rest": + continue + for value_index, value in enumerate( + self.space.property_values(statement.comp, property_index) + ): + if value == statement.rest: + continue + scored.append( + ( + float(action_lp[ACTION_SET_PROPERTY]) + + base + + float(property_value_lp[value_index]), + Edit( + ACTION_SET_PROPERTY, + stmt, + comp=property_index, + target=value_index, + ), + ) + ) for payload in range(len(V05_TEMPLATES)): scored.append( ( @@ -1308,7 +1450,7 @@ def _decode_one( } for row, (_, statements, _) in enumerate(live): candidates = self._enumerate_edits( - out, row, len(statements), len(inventory) + out, row, len(statements), len(inventory), statements=statements ) expanded = 0 for _, edit in candidates: diff --git a/src/slm_training/resources/variant_registry.json b/src/slm_training/resources/variant_registry.json index 1ea7a0b0b..3b61a7a91 100644 --- a/src/slm_training/resources/variant_registry.json +++ b/src/slm_training/resources/variant_registry.json @@ -1,6 +1,6 @@ { "count": 3, - "fingerprint": "2a1a10b9b397ff07ec4eb8330c8b57c3589a7d23087ea676ca74e836f9aa544a", + "fingerprint": "cd09cb02571e27c1cd6cf952a3240ec9ee835110213c99ad2cbd45ebdb209209", "schema": "variant_contract/v1", "variants": [ { @@ -15,10 +15,12 @@ "variant_id": "repl_operators" }, { - "action_alphabet_fingerprint": "ab2662a497d8359ffaee46ebbd4bee3789f5b0f2accaf8bf46c5dee489622dab", + "action_alphabet_fingerprint": "1de83afe8bc7db333bf0cac040a19fa355c34d28653a508343e45f136a1dac4c", "action_alphabet_id": "tree_edit_diffusion.edit_actions", "inventory_source": "pack", - "kernel_ops": [], + "kernel_ops": [ + "openui.set_property" + ], "pack_id": "openui", "schema": "variant_contract/v1", "seed_policy_id": "tree_edit_diffusion.minimal_valid_program_seed", diff --git a/src/slm_training/resources/versions.json b/src/slm_training/resources/versions.json index c1687677c..6e7f92ab4 100644 --- a/src/slm_training/resources/versions.json +++ b/src/slm_training/resources/versions.json @@ -5275,7 +5275,7 @@ ] }, "harness.experiments.slm299_edit_reachability": { - "version": "v5", + "version": "v6", "kind": "harness", "paths": [ "src/slm_training/harnesses/experiments/slm299_edit_reachability.py", @@ -5288,9 +5288,16 @@ "docs/design/iter-slm305-edit-language-20260724.md", "tests/test_models/test_tree_edit_diffusion.py", "docs/design/var0-03-pack-derived-inventory-20260726.json", - "docs/design/var0-03-pack-derived-inventory-20260726.md" + "docs/design/var0-03-pack-derived-inventory-20260726.md", + "docs/design/var1-02-set-property-action-20260726.json", + "docs/design/var1-02-set-property-action-20260726.md" ], "history": [ + { + "version": "v6", + "date": "2026-07-26", + "note": "SLM-425: add the real pack-owned SET_PROPERTY action to TreeEditSpace, its format-3 migration and SLM-299 production enumeration. The finite declared rest domain remains narrower than VAR1-01's hypothetical row control; the local audit is capability evidence only." + }, { "version": "v5", "date": "2026-07-26", @@ -7469,7 +7476,7 @@ ] }, "model.twotower": { - "version": "v255", + "version": "v256", "kind": "harness", "paths": [ "src/slm_training/models/twotower.py", @@ -7511,6 +7518,11 @@ "src/slm_training/resources/decode/speculative_ngram_v1.json" ], "history": [ + { + "version": "v256", + "date": "2026-07-26", + "note": "SLM-425: migrate tree-edit checkpoints from format 2 to 3 while preserving the existing action-head prefix and initializing the new property-value head." + }, { "version": "v255", "date": "2026-07-26", @@ -9300,7 +9312,7 @@ ] }, "dsl.variants": { - "version": "v2", + "version": "v3", "kind": "harness", "paths": [ "src/slm_training/dsl/variants.py", @@ -9308,6 +9320,11 @@ "tests/test_dsl/test_variants.py" ], "history": [ + { + "version": "v3", + "date": "2026-07-26", + "note": "SLM-425: tree_edit_diffusion declares openui.set_property as its realized kernel operation and regenerates the derived contract registry." + }, { "version": "v2", "date": "2026-07-26", diff --git a/tests/test_harnesses/experiments/test_slm299_edit_reachability.py b/tests/test_harnesses/experiments/test_slm299_edit_reachability.py index 9d9aba6c9..ecb65d44c 100644 --- a/tests/test_harnesses/experiments/test_slm299_edit_reachability.py +++ b/tests/test_harnesses/experiments/test_slm299_edit_reachability.py @@ -66,6 +66,23 @@ def test_seed_identity_is_zero_edits() -> None: assert case.edit_lower_bound == 0 +def test_extended_space_reaches_a_real_pack_property_mutation() -> None: + """SLM-425 makes the root's declared empty rest reachable in one edit.""" + target = "root = Stack([])" + case = analyze_reachability(SEED, target, slot_inventory=[":x"]) + assert case.verdict is Verdict.PROVEN_REACHABLE + assert case.edit_lower_bound == 1 + assert case.path == [ + { + "action": "SET_PROPERTY", + "stmt": 0, + "property": "rest", + "rest": "", + } + ] + + + # (b) multi-container target ---------------------------------------------- def test_multi_container_target_needs_container_add() -> None: diff --git a/tests/test_models/test_tree_edit_diffusion.py b/tests/test_models/test_tree_edit_diffusion.py index 02bc18498..183a52d13 100644 --- a/tests/test_models/test_tree_edit_diffusion.py +++ b/tests/test_models/test_tree_edit_diffusion.py @@ -66,6 +66,7 @@ def test_mutations_preserve_validity_and_inverse_restores() -> None: ACTION_REMOVE_CONTAINER, ACTION_REPLACE_STATEMENT, ACTION_REPLACE_SUBTREE, + ACTION_SET_PROPERTY, Edit, ) @@ -131,6 +132,64 @@ def apply(edit: Edit, stmts=None): assert space.apply(base, Edit(ACTION_REPLACE_STATEMENT, 0, payload=0), INVENTORY) is None +def test_set_property_round_trips_root_and_non_root_and_fails_closed() -> None: + """SLM-425 mutates a pack-owned container property, including root.""" + space = TreeEditSpace() + base = parse_statements(PROGRAM) + assert base is not None + + # The default pack's ``rest`` domain is (column, empty), so property 0 + # and target 1 select the empty form on the root. + root_mutation = Edit(ACTION_SET_PROPERTY, 0, comp=0, target=1) + root_changed = space.apply(base, root_mutation, INVENTORY) + assert root_changed is not None + assert root_changed[0].rest == "" + root_restored = space.apply( + root_changed, Edit(ACTION_SET_PROPERTY, 0, comp=0, target=0), INVENTORY + ) + assert root_restored is not None + assert render_statements(root_restored) == PROGRAM + + # The same structured rebuild works on an existing non-root container. + child_mutation = Edit(ACTION_SET_PROPERTY, 1, comp=0, target=0) + child_changed = space.apply(base, child_mutation, INVENTORY) + assert child_changed is not None + assert child_changed[1].rest == ', "column"' + child_restored = space.apply( + child_changed, Edit(ACTION_SET_PROPERTY, 1, comp=0, target=1), INVENTORY + ) + assert child_restored is not None + assert render_statements(child_restored) == PROGRAM + + # No undeclared property/value can silently mutate the source. + assert space.apply(base, Edit(ACTION_SET_PROPERTY, 0, comp=1, target=0), INVENTORY) is None + assert space.apply(base, Edit(ACTION_SET_PROPERTY, 0, comp=0, target=3), INVENTORY) is None + + +def test_set_property_rejects_a_pack_value_the_parser_does_not_accept() -> None: + """Pack metadata is an input domain, never a substitute for parsing.""" + import slm_training.dsl.pack as pack_mod + from slm_training.dsl.pack import get_pack + + base_pack = get_pack("openui") + custom = replace( + base_pack, + pack_id="tree-edit-invalid-property", + component_property_domains={ + **base_pack.component_property_domains, + "Stack": {"rest": (', "unterminated',)}, + }, + ) + pack_mod.register_pack(custom) + try: + space = TreeEditSpace(pack_id=custom.pack_id) + base = parse_statements(PROGRAM) + assert base is not None + assert space.apply(base, Edit(ACTION_SET_PROPERTY, 0), INVENTORY) is None + finally: + pack_mod._PACKS.pop(custom.pack_id, None) + + def test_extended_sample_mutation_loop_restores() -> None: space = TreeEditSpace() rng = random.Random(11) @@ -196,29 +255,32 @@ def test_checkpoint_format2_fail_closed_and_migration(tmp_path) -> None: model = TreeEditDiffusionModel.from_records(records, config=cfg, device="cpu") path = tmp_path / "ckpt.pt" model.save(path) - # Round-trip at format 2. + # Round-trip at the current format. loaded = TreeEditDiffusionModel.from_checkpoint(path, device="cpu") assert loaded.policy.action_head.out_features == model.policy.action_head.out_features - # Simulate a format-1 checkpoint: shrink the action head to 4 rows. + # Simulate a format-2 checkpoint: shrink the action head to the prior 11 + # actions and remove the SLM-425 property-value head. import torch as _torch payload = _torch.load(path, map_location="cpu", weights_only=False) - payload["format_version"] = 1 + payload["format_version"] = 2 sd = payload["state_dict"] - old_w = sd["policy.action_head.weight"][:4].clone() - old_b = sd["policy.action_head.bias"][:4].clone() + old_w = sd["policy.action_head.weight"][:11].clone() + old_b = sd["policy.action_head.bias"][:11].clone() sd["policy.action_head.weight"] = old_w sd["policy.action_head.bias"] = old_b - old_path = tmp_path / "ckpt_v1.pt" + del sd["policy.property_value_head.weight"] + del sd["policy.property_value_head.bias"] + old_path = tmp_path / "ckpt_v2.pt" _torch.save(payload, old_path) - (tmp_path / "ckpt_v1.tokenizer.json").write_text( + (tmp_path / "ckpt_v2.tokenizer.json").write_text( (tmp_path / "ckpt.tokenizer.json").read_text(encoding="utf-8"), encoding="utf-8", ) # Unmigrated old checkpoint fails closed with a clear error. - with pytest.raises(ValueError, match="format_version=1"): + with pytest.raises(ValueError, match="format_version=2"): TreeEditDiffusionModel.from_checkpoint(old_path, device="cpu") # Migration warm-starts: old action rows preserved, new rows initialized. @@ -228,15 +290,26 @@ def test_checkpoint_format2_fail_closed_and_migration(tmp_path) -> None: report = migrate_tree_edit_checkpoint( source_checkpoint=old_path, output_checkpoint=out_path ) - assert report["source_format_version"] == 1 - assert report["output_format_version"] == 2 - assert report["preserved_action_head_rows"] == 4 + assert report["source_format_version"] == 2 + assert report["output_format_version"] == 3 + assert report["preserved_action_head_rows"] == 11 assert (tmp_path / "ckpt_migrated.migrate.json").exists() migrated = TreeEditDiffusionModel.from_checkpoint(out_path, device="cpu") new_w = migrated.policy.action_head.weight assert new_w.shape[0] == model.policy.action_head.weight.shape[0] - assert _torch.allclose(new_w[:4], old_w) - assert _torch.allclose(migrated.policy.action_head.bias[:4], old_b) + assert _torch.allclose(new_w[:11], old_w) + assert _torch.allclose(migrated.policy.action_head.bias[:11], old_b) + assert migrated.policy.property_value_head.out_features == 2 + + # Existing action logits are bit-identical after the format migration. + state = model._state_batch([PROGRAM]) + ctx = _torch.zeros((1, 1, cfg.d_model)) + ctx_pad = _torch.zeros((1, 1), dtype=_torch.bool) + model.eval() + migrated.eval() + before = model.policy(state, model.tokenizer.pad_id, ctx, ctx_pad)["action"] + after = migrated.policy(state, migrated.tokenizer.pad_id, ctx, ctx_pad)["action"] + assert _torch.equal(before[:, :11], after[:, :11]) def test_training_loss_decode_all_valid_and_checkpoint(tmp_path) -> None: