From d56223b45eb974c08f3f955e89921b3d38a9980f Mon Sep 17 00:00:00 2001 From: Yernat Yestekov <2068106+doublewhy@users.noreply.github.com> Date: Fri, 9 Oct 2026 02:02:47 -0700 Subject: [PATCH 01/11] feat(examples): add machine-checked metadata to example library entries Every worked example, template, and pattern entry in examples/library/catalog.yaml now records validation_status (validated or guidance), sdl_sections, intended_user, and limits. The gate rejects an entry whose status is outside the vocabulary, a template that is not validated, a pattern that claims validation, an unknown intended user, empty limits, or a section name that is not a current SDL section. Validated worked examples are now parsed through parse_sdl_file and fail on errors or advisories, and a validated entry must list only sections present in its SDL. The catalog description no longer calls every entry validated. The entry checks live in a new support module, tools/example_library_checks.py, because tools/check_example_library.py was already 488 lines against SonarCloud's 500-line file limit. The template-body SDL check moves there with its rule ids and messages. The module imports raes at module level, so the example-library-template-import rule for a failed raes import is removed. The top-level fields that are not sections come from tools/sdl_catalog_parity/_paths.py, which the SDL catalog parity gate keeps in step with specs/sdl/sections.md. examples/README.md documents the fields and what the gate checks. --- examples/README.md | 25 +++- examples/library/catalog.yaml | 99 ++++++++++++- .../tests/test_example_library_policy.py | 26 +++- ...test_issue_381_example_catalog_metadata.py | 121 ++++++++++++++++ tools/check_example_library.py | 35 ++--- tools/example_library_checks.py | 136 ++++++++++++++++++ 6 files changed, 410 insertions(+), 32 deletions(-) create mode 100644 implementations/python/tests/test_issue_381_example_catalog_metadata.py create mode 100644 tools/example_library_checks.py diff --git a/examples/README.md b/examples/README.md index aea9fba4d..9b92dd774 100644 --- a/examples/README.md +++ b/examples/README.md @@ -84,8 +84,29 @@ machine-readable catalog for the current non-normative authoring library. Each template has metadata plus a complete current-SDL `body`. The `tools/check_example_library.py` policy gate validates catalog shape, stable -IDs, referenced paths, AUT-806 requirement references, and every template body -through the SDL parser and semantic validator. +IDs, referenced paths, AUT-806 requirement references, entry metadata, and +every template body and validated worked example through the SDL parser and +semantic validator. + +### Catalog entry metadata + +Every `worked_examples`, `templates`, and `patterns` entry in the catalog +records four fields. The policy gate rejects an entry that omits a field or +breaks these rules. + +| Field | Allowed values | What the gate checks | +|-------|----------------|----------------------| +| `validation_status` | `validated` or `guidance` | For `validated`, the gate parses the template body or worked-example file with the SDL parser and semantic validator, and fails on any error or advisory. A `guidance` file is not parsed as SDL. Templates must be `validated`; patterns must be `guidance`. | +| `sdl_sections` | Top-level section names from [`../specs/sdl/sections.md`](../specs/sdl/sections.md), such as `nodes` or `workflows` | Each name is a current SDL section, not a metadata or composition field such as `name` or `imports`. For a `validated` entry, each listed section is present in the validated SDL. | +| `intended_user` | `sdl-author` | The value is in the allowed list. | +| `limits` | One or more short statements of what the entry does not show | The list is present and not empty. Reviewers check the wording. | + +`sdl_sections` names the sections that an entry shows for its surface, so a +large worked example lists only some of the sections it uses. `validated` +means that current SDL validation accepts the file or template body. It does +not mean that a backend can realize the scenario or that anything has run. A +`guidance` entry is reading material, such as a prose pattern or a test +module, and is not executable SDL. ## Validate The Examples diff --git a/examples/library/catalog.yaml b/examples/library/catalog.yaml index 4af55e53b..43d981fb6 100644 --- a/examples/library/catalog.yaml +++ b/examples/library/catalog.yaml @@ -6,8 +6,8 @@ source_refs: - docs/public/quickstart.md - docs/explain/reference/canonical-reference-map.md description: > - Domain-neutral RAES agentic-environment authoring library for non-normative, - validated examples, templates, and reusable patterns. Current examples are + Domain-neutral RAES agentic-environment authoring library for non-normative + examples, templates, and reusable patterns. Current examples are strongest in cyber; additional domains extend through their own profiles, assets, backends, and evidence requirements. Template bodies use current SDL so they can be parsed and semantically validated by the Python reference @@ -19,72 +19,167 @@ surfaces: - id: hospital-ransomware-surgery-day path: examples/scenarios/hospital-ransomware-surgery-day.sdl.yaml source_refs: [examples/README.md] + validation_status: validated + sdl_sections: [nodes, infrastructure, features, entities, agents, relationships, content, conditions, objectives] + intended_user: sdl-author + limits: + - Does not deploy a hospital range or prove clinical exercise adequacy. - id: cross-backend-minimal path: examples/scenarios/cross-backend-minimal.sdl.yaml source_refs: [examples/README.md] + validation_status: validated + sdl_sections: [nodes, infrastructure] + intended_user: sdl-author + limits: + - Does not exercise an OCI runtime or live libvirt/QEMU daemon and makes no infrastructure-equivalence or fidelity claim. templates: - id: minimal-validated-scenario path: examples/library/templates/scenario/minimal-validated-scenario.yaml + validation_status: validated + sdl_sections: [nodes, conditions, entities, agents, propositions, assertions, evidence_requirements, objectives, workflows] + intended_user: sdl-author + limits: + - Validation covers SDL structure and references; it does not show that a backend can realize the scenario. patterns: - id: scenario-reference-integrity path: examples/library/patterns/scenario-reference-integrity.yaml + validation_status: guidance + sdl_sections: [nodes, entities, agents, objectives, workflows] + intended_user: sdl-author + limits: + - Covers reference resolution only, not backend realization. workflow: summary: Declarative workflow authoring over objectives and branch state. worked_examples: - id: satcom-release-poisoning-workflows path: examples/scenarios/satcom-release-poisoning.sdl.yaml source_refs: [examples/README.md] + validation_status: validated + sdl_sections: [objectives, workflows, variables] + intended_user: sdl-author + limits: + - Does not implement a CI/CD backend or production release system. templates: - id: parallel-objective-workflow path: examples/library/templates/workflow/parallel-objective-workflow.yaml + validation_status: validated + sdl_sections: [nodes, conditions, entities, agents, propositions, assertions, evidence_requirements, objectives, workflows] + intended_user: sdl-author + limits: + - Validation covers the workflow graph and its references; no workflow is executed. patterns: - id: workflow-explicit-control-graph path: examples/library/patterns/workflow-explicit-control-graph.yaml + validation_status: guidance + sdl_sections: [objectives, workflows] + intended_user: sdl-author + limits: + - Covers workflow graph authoring only; it does not execute workflows. participant_behavior: summary: Participant action contracts, observation boundaries, behavior specifications, and agent bindings. worked_examples: - id: sem-208-participant-behavior path: implementations/python/tests/test_sem_208_participant_behavior.py source_refs: [docs/explain/sdl/validation.md] + validation_status: guidance + sdl_sections: [agents, action_contracts, observation_boundaries, behavior_specifications] + intended_user: sdl-author + limits: + - A pytest module rather than an SDL file; the test suite runs its inline SDL, this gate does not. templates: - id: action-contract-observation-boundary path: examples/library/templates/participant_behavior/action-contract-observation-boundary.yaml + validation_status: validated + sdl_sections: [nodes, entities, action_contracts, observation_boundaries, agents, behavior_specifications] + intended_user: sdl-author + limits: + - Validation covers the participant declarations and their references; it does not run a participant or show backend support. patterns: - id: participant-behavior-contract-binding path: examples/library/patterns/participant-behavior-contract-binding.yaml + validation_status: guidance + sdl_sections: [agents, action_contracts, observation_boundaries] + intended_user: sdl-author + limits: + - Covers declaring and binding contracts only; it does not show participant runtime behavior. task: summary: Task-oriented authoring using current objectives, conditions, and target refs. worked_examples: - id: port-authority-objective-tasks path: examples/scenarios/port-authority-surge-response.sdl.yaml source_refs: [examples/README.md] + validation_status: validated + sdl_sections: [agents, conditions, propositions, assertions, objectives, workflows] + intended_user: sdl-author + limits: + - Does not implement OT control, safety validation, or port operations. templates: - id: single-objective-task path: examples/library/templates/task/single-objective-task.yaml + validation_status: validated + sdl_sections: [nodes, conditions, entities, agents, propositions, assertions, evidence_requirements, objectives, workflows] + intended_user: sdl-author + limits: + - SDL has no first-class tasks section, and this objective is not an experiment-task-v1 record. patterns: - id: task-as-objective-contract path: examples/library/patterns/task-as-objective-contract.yaml + validation_status: guidance + sdl_sections: [agents, conditions, objectives, workflows] + intended_user: sdl-author + limits: + - SDL has no first-class tasks section, and an objective written this way is not an experiment-task-v1 record. run: summary: Run-oriented authoring with story, script, event, workflow, and objective windows. worked_examples: - id: hospital-run-story path: examples/scenarios/hospital-ransomware-surgery-day.sdl.yaml source_refs: [examples/README.md] + validation_status: validated + sdl_sections: [injects, events, scripts, stories, objectives] + intended_user: sdl-author + limits: + - Story, script, and event timing is authored SDL; nothing is scheduled or run. templates: - id: timed-run-control path: examples/library/templates/run/timed-run-control.yaml + validation_status: validated + sdl_sections: [nodes, conditions, injects, events, scripts, stories, entities, agents, propositions, assertions, evidence_requirements, objectives, workflows] + intended_user: sdl-author + limits: + - SDL has no first-class runs section; the template expresses a run with stories, scripts, and events, and nothing is scheduled or run. patterns: - id: run-window-with-evidence path: examples/library/patterns/run-window-with-evidence.yaml + validation_status: guidance + sdl_sections: [injects, events, scripts, stories, objectives, workflows] + intended_user: sdl-author + limits: + - SDL has no first-class runs section; the pattern maps a run onto stories, scripts, events, and objective windows. study: summary: Study-oriented authoring with observable conditions, objectives, and workflow protocol. Graded scoring is an experiment/evaluator-plane concern per ADR-073. worked_examples: - id: satcom-study-scenario path: examples/scenarios/satcom-release-poisoning.sdl.yaml source_refs: [examples/README.md] + validation_status: validated + sdl_sections: [conditions, propositions, assertions, evidence_requirements, objectives, workflows] + intended_user: sdl-author + limits: + - Declares observable conditions only; graded scoring and reward are outside the SDL per ADR-073. templates: - id: observational-study-protocol path: examples/library/templates/study/observational-study-protocol.yaml + validation_status: validated + sdl_sections: [nodes, conditions, entities, agents, propositions, assertions, evidence_requirements, objectives, workflows] + intended_user: sdl-author + limits: + - SDL has no first-class studies section; graded scoring stays in the experiment/evaluator plane per ADR-073. patterns: - id: observable-study-conditions path: examples/library/patterns/observable-study-conditions.yaml + validation_status: guidance + sdl_sections: [conditions, propositions, assertions, evidence_requirements, objectives, workflows] + intended_user: sdl-author + limits: + - SDL has no first-class studies section; graded scoring stays in the experiment/evaluator plane per ADR-073. diff --git a/implementations/python/tests/test_example_library_policy.py b/implementations/python/tests/test_example_library_policy.py index 41a39afaa..ff4c26100 100644 --- a/implementations/python/tests/test_example_library_policy.py +++ b/implementations/python/tests/test_example_library_policy.py @@ -102,6 +102,15 @@ def _write_yaml(path: Path, value: dict[str, Any]) -> None: path.write_text(yaml.safe_dump(value, sort_keys=False), encoding="utf-8") +def _entry_metadata(validation_status: str, sdl_sections: list[str]) -> dict[str, Any]: + return { + "validation_status": validation_status, + "sdl_sections": sdl_sections, + "intended_user": "sdl-author", + "limits": ["Policy-test fixture; it makes no support claim."], + } + + def _seed_repo(tmp_path: Path) -> Path: catalog: dict[str, Any] = { "library": "raes-example-pattern-library", @@ -121,10 +130,23 @@ def _seed_repo(tmp_path: Path) -> Path: "id": f"{surface}-worked", "path": worked_path, "source_refs": ["examples/README.md"], + **_entry_metadata("guidance", ["objectives"]), + } + ], + "templates": [ + { + "id": f"{surface}-template", + "path": template_path, + **_entry_metadata("validated", ["nodes", "objectives", "workflows"]), + } + ], + "patterns": [ + { + "id": f"{surface}-pattern", + "path": pattern_path, + **_entry_metadata("guidance", ["objectives"]), } ], - "templates": [{"id": f"{surface}-template", "path": template_path}], - "patterns": [{"id": f"{surface}-pattern", "path": pattern_path}], } worked_file = tmp_path / worked_path worked_file.parent.mkdir(parents=True, exist_ok=True) diff --git a/implementations/python/tests/test_issue_381_example_catalog_metadata.py b/implementations/python/tests/test_issue_381_example_catalog_metadata.py new file mode 100644 index 000000000..150fad184 --- /dev/null +++ b/implementations/python/tests/test_issue_381_example_catalog_metadata.py @@ -0,0 +1,121 @@ +"""Catalog entry metadata rules for the example library (issue #381).""" + +from __future__ import annotations + +from collections.abc import Callable +from pathlib import Path +from typing import Any + +import pytest +import yaml +from test_example_library_policy import _VALID_BODY, _read_catalog, _seed_repo, _write_catalog +from tools.check_example_library import evaluate_example_library + +_Mutation = Callable[[dict[str, Any]], None] +_WORKED_SDL_PATH = "examples/scenarios/worked.sdl.yaml" +_VALID_SDL = yaml.safe_dump(_VALID_BODY, sort_keys=False).encode("utf-8") + + +def _set(field: str, key: str, value: Any) -> _Mutation: + def mutate(catalog: dict[str, Any]) -> None: + catalog["surfaces"]["scenario"][field][0][key] = value + + return mutate + + +def _rule_ids(repo_root: Path) -> list[str]: + return [failure.rule_id for failure in evaluate_example_library(repo_root)] + + +@pytest.mark.parametrize( + ("mutation", "rule_id"), + ( + pytest.param( + _set("worked_examples", "validation_status", "supported"), + "example-library-entry-status", + id="status-outside-vocabulary", + ), + pytest.param( + _set("templates", "validation_status", "guidance"), + "example-library-entry-status", + id="template-not-validated", + ), + pytest.param( + _set("patterns", "validation_status", "validated"), + "example-library-entry-status", + id="pattern-claims-validation", + ), + pytest.param(_set("templates", "intended_user", "operator"), "example-library-entry-user", id="unknown-user"), + pytest.param(_set("patterns", "limits", []), "example-library-entry-limits", id="empty-limits"), + pytest.param( + _set("patterns", "limits", "One statement."), "example-library-entry-limits", id="limits-not-a-list" + ), + pytest.param(_set("patterns", "sdl_sections", ["tasks"]), "example-library-entry-sections", id="not-sdl"), + pytest.param(_set("patterns", "sdl_sections", ["name"]), "example-library-entry-sections", id="metadata-field"), + pytest.param( + _set("worked_examples", "sdl_sections", ["objectives", "objectives"]), + "example-library-entry-sections", + id="duplicate-section", + ), + pytest.param( + _set("templates", "sdl_sections", ["nodes", "stories"]), + "example-library-entry-sections", + id="section-absent-from-template-body", + ), + pytest.param( + _set("templates", "sdl_sections", ["nodes", "tasks"]), + "example-library-entry-sections", + id="not-sdl-on-validated-template", + ), + pytest.param( + _set("worked_examples", "path", "examples/library/worked/missing.txt"), + "example-library-path-missing", + id="worked-example-path-missing", + ), + ), +) +def test_catalog_entry_violation_is_the_only_failure(tmp_path: Path, mutation: _Mutation, rule_id: str) -> None: + repo_root = _seed_repo(tmp_path) + catalog = _read_catalog(repo_root) + mutation(catalog) + _write_catalog(repo_root, catalog) + + assert _rule_ids(repo_root) == [rule_id] + + +@pytest.mark.parametrize( + ("sdl_source", "sdl_sections", "expected"), + ( + pytest.param(_VALID_SDL, ["nodes", "workflows"], [], id="valid"), + pytest.param(_VALID_SDL, ["stories"], ["example-library-entry-sections"], id="section-absent-from-file"), + pytest.param( + b"name: invalid\nnodes: [not-a-map]\n", + ["nodes"], + ["example-library-worked-example-body"], + id="invalid-sdl", + ), + pytest.param( + b"name: advisory\nnodes:\n vm:\n type: compute\n", + ["nodes"], + ["example-library-worked-example-advisory"], + id="advisory", + ), + pytest.param(b"name: \xff\n", ["nodes"], ["example-library-worked-example-body"], id="not-utf-8"), + ), +) +def test_validated_worked_example_is_parsed_as_sdl( + tmp_path: Path, sdl_source: bytes, sdl_sections: list[str], expected: list[str] +) -> None: + repo_root = _seed_repo(tmp_path) + worked_file = repo_root / _WORKED_SDL_PATH + worked_file.parent.mkdir(parents=True) + worked_file.write_bytes(sdl_source) + catalog = _read_catalog(repo_root) + catalog["surfaces"]["scenario"]["worked_examples"][0].update( + path=_WORKED_SDL_PATH, + validation_status="validated", + sdl_sections=sdl_sections, + ) + _write_catalog(repo_root, catalog) + + assert _rule_ids(repo_root) == expected diff --git a/tools/check_example_library.py b/tools/check_example_library.py index 095f01064..54e4dabd0 100644 --- a/tools/check_example_library.py +++ b/tools/check_example_library.py @@ -16,6 +16,7 @@ import yaml +from tools.example_library_checks import check_entry_metadata, check_template_body, check_worked_example_file from tools.policy.common import PolicyFailure, apply_exceptions, failures_to_json, load_exceptions CATALOG_RELATIVE_PATH = "examples/library/catalog.yaml" @@ -256,6 +257,7 @@ def _check_reference_entry( ) failures.extend(_check_unique_id(entry.get("id"), f"surfaces.{surface}.{field}[{index}]", seen_ids)) + failures.extend(check_entry_metadata(field, entry, f"surfaces.{surface}.{field}[{index}]")) absolute, relative, path_failure = _repo_relative_path( entry.get("path"), repo_root=repo_root, @@ -263,8 +265,8 @@ def _check_reference_entry( ) if path_failure is not None: failures.append(path_failure) - elif absolute is not None and relative is not None and not absolute.is_file(): - failures.append(_fail("example-library-path-missing", f"referenced path does not exist: {relative}", relative)) + elif absolute is not None and relative is not None: + failures.extend(check_worked_example_file(absolute, relative, entry)) if "source_refs" in entry and _str_list(entry["source_refs"]) is None: failures.append( @@ -305,6 +307,7 @@ def _check_artifact_entries( ) entry_id = entry.get("id") failures.extend(_check_unique_id(entry_id, f"surfaces.{surface}.{field}[{index}]", seen_ids)) + failures.extend(check_entry_metadata(field, entry, f"surfaces.{surface}.{field}[{index}]")) absolute, relative, path_failure = _repo_relative_path( entry.get("path"), repo_root=repo_root, @@ -321,7 +324,7 @@ def _check_artifact_entries( ) continue if artifact_kind == "template": - failures.extend(_check_template_file(absolute, relative, surface=surface, catalog_id=entry_id)) + failures.extend(_check_template_file(absolute, relative, surface=surface, entry=entry)) else: failures.extend(_check_pattern_file(absolute, relative, surface=surface, catalog_id=entry_id)) return failures @@ -336,10 +339,11 @@ def _check_unique_id(value: Any, owner: str, seen_ids: set[str]) -> list[PolicyF return [] -def _check_template_file(path: Path, relative_path: str, *, surface: str, catalog_id: Any) -> list[PolicyFailure]: +def _check_template_file(path: Path, relative_path: str, *, surface: str, entry: dict[str, Any]) -> list[PolicyFailure]: raw, failures = _load_yaml_file(path, relative_path) if raw is None: return failures + catalog_id = entry.get("id") failures.extend(_check_artifact_common(raw, relative_path, surface=surface, catalog_id=catalog_id, kind="template")) if "template" in raw and raw["template"] != TEMPLATE_VALUE: @@ -354,7 +358,7 @@ def _check_template_file(path: Path, relative_path: str, *, surface: str, catalo if not isinstance(body, dict): failures.append(_fail("example-library-template-body", "template body must be a mapping", relative_path)) return failures - failures.extend(_check_template_body(body, relative_path)) + failures.extend(check_template_body(entry, body, relative_path)) return failures @@ -426,27 +430,6 @@ def _check_artifact_common( return failures -def _check_template_body(body: dict[str, Any], relative_path: str) -> list[PolicyFailure]: - try: - from raes import parse_sdl - except ImportError as exc: # pragma: no cover - policy runs inside the project environment - return [_fail("example-library-template-import", f"could not import raes: {exc}", relative_path)] - - try: - scenario = parse_sdl(yaml.safe_dump(body, sort_keys=False)) - except Exception as exc: # noqa: BLE001 - render parser/validator failures as policy failures - return [_fail("example-library-template-body", f"template body is not valid SDL: {exc}", relative_path)] - if scenario.advisories: - return [ - _fail( - "example-library-template-advisory", - "template body produced advisories: " + "; ".join(scenario.advisories), - relative_path, - ) - ] - return [] - - def evaluate_example_library(repo_root: Path = REPO_ROOT) -> list[PolicyFailure]: raw, failures = _load_yaml_file(repo_root / CATALOG_RELATIVE_PATH, CATALOG_RELATIVE_PATH) if raw is None: diff --git a/tools/example_library_checks.py b/tools/example_library_checks.py new file mode 100644 index 000000000..894b54038 --- /dev/null +++ b/tools/example_library_checks.py @@ -0,0 +1,136 @@ +"""Entry metadata and validated-content checks for the AUT-806 example library gate.""" + +from __future__ import annotations + +from functools import cache +from pathlib import Path +from typing import Any + +import yaml +from raes import Scenario, parse_sdl, parse_sdl_file + +from tools.policy.common import PolicyFailure +from tools.sdl_catalog_parity._paths import _COMPOSITION_FIELDS, _METADATA_FIELDS + +CATALOG_RELATIVE_PATH = "examples/library/catalog.yaml" +VALIDATED = "validated" +GUIDANCE = "guidance" +VALIDATION_STATUSES: tuple[str, ...] = (VALIDATED, GUIDANCE) +REQUIRED_VALIDATION_STATUS: dict[str, str] = { + "templates": VALIDATED, + "patterns": GUIDANCE, +} +INTENDED_USERS: tuple[str, ...] = ("sdl-author",) +# Top-level SDL fields that specs/sdl/sections.md classifies as metadata or +# composition rather than as sections. The SDL catalog parity gate keeps these +# sets in step with that catalog. +NON_SECTION_FIELDS = _METADATA_FIELDS | _COMPOSITION_FIELDS + + +def _fail(rule_id: str, message: str, path: str = CATALOG_RELATIVE_PATH) -> PolicyFailure: + return PolicyFailure(rule_id, message, path) + + +def _text_list(value: object) -> list[str] | None: + if isinstance(value, list) and all(isinstance(item, str) and item for item in value): + return list(value) + return None + + +@cache +def _sdl_section_names() -> frozenset[str]: + return frozenset(Scenario.model_fields) - NON_SECTION_FIELDS + + +def check_entry_metadata(field: str, entry: dict[str, Any], owner: str) -> list[PolicyFailure]: + """Check the validation status, SDL sections, intended user, and limits of one catalog entry.""" + failures: list[PolicyFailure] = [] + allowed_statuses = ( + (REQUIRED_VALIDATION_STATUS[field],) if field in REQUIRED_VALIDATION_STATUS else VALIDATION_STATUSES + ) + status = entry.get("validation_status") + if status not in allowed_statuses: + failures.append( + _fail( + "example-library-entry-status", + f"{owner}.validation_status must be one of {list(allowed_statuses)}; got {status!r}", + ) + ) + user = entry.get("intended_user") + if user not in INTENDED_USERS: + failures.append( + _fail( + "example-library-entry-user", + f"{owner}.intended_user must be one of {list(INTENDED_USERS)}; got {user!r}", + ) + ) + if not _text_list(entry.get("limits")): + failures.append(_fail("example-library-entry-limits", f"{owner}.limits must be a non-empty list of strings")) + sections = _text_list(entry.get("sdl_sections")) + if not sections or len(set(sections)) != len(sections): + failures.append( + _fail("example-library-entry-sections", f"{owner}.sdl_sections must be a non-empty list of distinct names") + ) + elif unknown := sorted(set(sections) - _sdl_section_names()): + failures.append( + _fail( + "example-library-entry-sections", + f"{owner}.sdl_sections names values that are not SDL sections: {unknown}", + ) + ) + return failures + + +def _check_declared_sections( + entry: dict[str, Any], document: dict[str, Any], relative_path: str +) -> list[PolicyFailure]: + # Names that are not SDL sections already fail check_entry_metadata. + declared = set(_text_list(entry.get("sdl_sections")) or []) & _sdl_section_names() + absent = sorted(declared - set(document)) + if absent: + return [ + _fail( + "example-library-entry-sections", + f"sdl_sections lists sections absent from the validated SDL: {absent}", + relative_path, + ) + ] + return [] + + +def _check_sdl(source: str | Path, relative_path: str, *, kind: str, subject: str) -> list[PolicyFailure]: + """Parse SDL text, or an SDL file through the file boundary, and fail on errors or advisories.""" + try: + scenario = parse_sdl_file(source) if isinstance(source, Path) else parse_sdl(source) + except Exception as exc: # noqa: BLE001 - render parser/validator failures as policy failures + return [_fail(f"example-library-{kind}-body", f"{subject} is not valid SDL: {exc}", relative_path)] + failures: list[PolicyFailure] = [] + if scenario.advisories: + failures.append( + _fail( + f"example-library-{kind}-advisory", + f"{subject} produced advisories: " + "; ".join(scenario.advisories), + relative_path, + ) + ) + return failures + + +def check_template_body(entry: dict[str, Any], body: dict[str, Any], relative_path: str) -> list[PolicyFailure]: + """Validate a template body as SDL and check the entry's declared sections against it.""" + failures = _check_sdl( + yaml.safe_dump(body, sort_keys=False), relative_path, kind="template", subject="template body" + ) + return failures or _check_declared_sections(entry, body, relative_path) + + +def check_worked_example_file(path: Path, relative_path: str, entry: dict[str, Any]) -> list[PolicyFailure]: + """Require the worked-example file and, for a validated entry, validate it as SDL.""" + if not path.is_file(): + return [ + _fail("example-library-path-missing", f"referenced path does not exist: {relative_path}", relative_path) + ] + if entry.get("validation_status") != VALIDATED: + return [] + failures = _check_sdl(path, relative_path, kind="worked-example", subject="worked example") + return failures or _check_declared_sections(entry, yaml.safe_load(path.read_text(encoding="utf-8")), relative_path) From 66588d6ff3a338c4739dbe75c09b3dd2a2469d4a Mon Sep 17 00:00:00 2001 From: Yernat Yestekov <2068106+doublewhy@users.noreply.github.com> Date: Sat, 10 Oct 2026 09:06:33 -0700 Subject: [PATCH 02/11] fix(examples): read validated worked-example sections with the SDL loader The section-presence check re-read each validated worked example with PyYAML's YAML 1.1 yaml.safe_load after parse_sdl_file had accepted it. sdl-yaml/v1 resolves plain scalars with YAML 1.2 Core, so valid SDL such as an unquoted "description: 2026-02-30" or "description: =" raised ValueError or ConstructorError there, and the policy gate exited with a traceback instead of returning failures. The check now loads the file with load_sdl_fragment, the loader that parse_sdl_file uses. The absent-section failure also names the catalog entry, because the hospital and satcom worked-example files are each referenced by two entries. Now that patterns are guidance, docs/explain/getting-started.md (the Current Boundary list and the Template And Pattern Boundary section) and the canonical reference map no longer call the whole library validated. The README says the template check runs on the body as PyYAML loads it, and that the gate does not parse a guidance entry as SDL. The task-as-objective-contract pattern no longer tells authors to add metrics, evaluations, TLOs, or goals, which ADR-073 removed from SDL, and its catalog entry now lists the propositions and assertions it shows. The sem-208 limit says the test suite parses and validates its inline SDL rather than running it. --- docs/explain/getting-started.md | 4 +- .../reference/canonical-reference-map.md | 2 +- examples/README.md | 4 +- examples/library/catalog.yaml | 4 +- .../patterns/task-as-objective-contract.yaml | 3 +- ...test_issue_381_example_catalog_metadata.py | 50 ++++++++++++++----- tools/example_library_checks.py | 9 ++-- 7 files changed, 53 insertions(+), 23 deletions(-) diff --git a/docs/explain/getting-started.md b/docs/explain/getting-started.md index be60b00c9..56c4ac352 100644 --- a/docs/explain/getting-started.md +++ b/docs/explain/getting-started.md @@ -38,7 +38,7 @@ RAES can currently support: - parsing and validating SDL through the Python implementation - instantiating variables and compiling runtime models in the reference stack - checking backend contract fixtures and conformance profiles -- browsing validated, non-normative examples, templates, and reusable patterns +- browsing non-normative examples, templates, and reusable patterns for scenarios, workflows, participant behavior, tasks, runs, and studies - reviewing the specifications, ADRs, and examples that define current claims @@ -219,7 +219,7 @@ Do not use examples to claim: ## Template And Pattern Boundary -The current library provides validated authoring aids, not new runtime +The current library provides authoring aids, not new runtime authority. Current status: diff --git a/docs/explain/reference/canonical-reference-map.md b/docs/explain/reference/canonical-reference-map.md index 3c3bba10a..00659c41a 100644 --- a/docs/explain/reference/canonical-reference-map.md +++ b/docs/explain/reference/canonical-reference-map.md @@ -94,7 +94,7 @@ page is an index, not a replacement for the linked artifacts. ## Current Materialization Notes - SDL authoring, parsing, validation, instantiation, compilation, planning, - runtime manager/control-plane APIs, published JSON schemas, and a validated + runtime manager/control-plane APIs, published JSON schemas, and a non-normative template/pattern library are present in the repository. - Participant-implementation manifests, evidence-capture contracts, and provenance contract surfaces are described at the architecture level and are diff --git a/examples/README.md b/examples/README.md index 9b92dd774..e35012ea7 100644 --- a/examples/README.md +++ b/examples/README.md @@ -96,7 +96,7 @@ breaks these rules. | Field | Allowed values | What the gate checks | |-------|----------------|----------------------| -| `validation_status` | `validated` or `guidance` | For `validated`, the gate parses the template body or worked-example file with the SDL parser and semantic validator, and fails on any error or advisory. A `guidance` file is not parsed as SDL. Templates must be `validated`; patterns must be `guidance`. | +| `validation_status` | `validated` or `guidance` | For `validated`, the gate runs the SDL parser and semantic validator on the worked-example file, or on the template body as PyYAML loads it from the template file, and fails on any error or advisory they report. A `guidance` file is not parsed as SDL. Templates must be `validated`; patterns must be `guidance`. | | `sdl_sections` | Top-level section names from [`../specs/sdl/sections.md`](../specs/sdl/sections.md), such as `nodes` or `workflows` | Each name is a current SDL section, not a metadata or composition field such as `name` or `imports`. For a `validated` entry, each listed section is present in the validated SDL. | | `intended_user` | `sdl-author` | The value is in the allowed list. | | `limits` | One or more short statements of what the entry does not show | The list is present and not empty. Reviewers check the wording. | @@ -106,7 +106,7 @@ large worked example lists only some of the sections it uses. `validated` means that current SDL validation accepts the file or template body. It does not mean that a backend can realize the scenario or that anything has run. A `guidance` entry is reading material, such as a prose pattern or a test -module, and is not executable SDL. +module, and the gate does not parse it as SDL. ## Validate The Examples diff --git a/examples/library/catalog.yaml b/examples/library/catalog.yaml index 43d981fb6..caa9b430f 100644 --- a/examples/library/catalog.yaml +++ b/examples/library/catalog.yaml @@ -85,7 +85,7 @@ surfaces: sdl_sections: [agents, action_contracts, observation_boundaries, behavior_specifications] intended_user: sdl-author limits: - - A pytest module rather than an SDL file; the test suite runs its inline SDL, this gate does not. + - A pytest module rather than an SDL file; the test suite parses and validates its inline SDL, and this gate does not. templates: - id: action-contract-observation-boundary path: examples/library/templates/participant_behavior/action-contract-observation-boundary.yaml @@ -125,7 +125,7 @@ surfaces: - id: task-as-objective-contract path: examples/library/patterns/task-as-objective-contract.yaml validation_status: guidance - sdl_sections: [agents, conditions, objectives, workflows] + sdl_sections: [agents, conditions, propositions, assertions, objectives, workflows] intended_user: sdl-author limits: - SDL has no first-class tasks section, and an objective written this way is not an experiment-task-v1 record. diff --git a/examples/library/patterns/task-as-objective-contract.yaml b/examples/library/patterns/task-as-objective-contract.yaml index 6b6a6d04a..de892a6fc 100644 --- a/examples/library/patterns/task-as-objective-contract.yaml +++ b/examples/library/patterns/task-as-objective-contract.yaml @@ -16,7 +16,8 @@ use_when: authoring_steps: - Create or reuse the participant agent that owns the task. - Define the task target as a targetable SDL reference. - - Add a condition, metric, evaluation, TLO, or goal that proves completion. + - Reference a postcondition assertion over a completion proposition from objective success. + - Bind a condition probe to the proposition when an executable check is needed. - Place the objective in a workflow when task order matters. validation: - command: parse_sdl_file diff --git a/implementations/python/tests/test_issue_381_example_catalog_metadata.py b/implementations/python/tests/test_issue_381_example_catalog_metadata.py index 150fad184..d503d48e2 100644 --- a/implementations/python/tests/test_issue_381_example_catalog_metadata.py +++ b/implementations/python/tests/test_issue_381_example_catalog_metadata.py @@ -27,6 +27,23 @@ def _rule_ids(repo_root: Path) -> list[str]: return [failure.rule_id for failure in evaluate_example_library(repo_root)] +def _seed_worked_sdl(tmp_path: Path, sdl_source: bytes, sections_by_surface: dict[str, list[str]]) -> Path: + """Point each named surface's worked example at one validated SDL file.""" + repo_root = _seed_repo(tmp_path) + worked_file = repo_root / _WORKED_SDL_PATH + worked_file.parent.mkdir(parents=True) + worked_file.write_bytes(sdl_source) + catalog = _read_catalog(repo_root) + for surface, sdl_sections in sections_by_surface.items(): + catalog["surfaces"][surface]["worked_examples"][0].update( + path=_WORKED_SDL_PATH, + validation_status="validated", + sdl_sections=sdl_sections, + ) + _write_catalog(repo_root, catalog) + return repo_root + + @pytest.mark.parametrize( ("mutation", "rule_id"), ( @@ -101,21 +118,30 @@ def test_catalog_entry_violation_is_the_only_failure(tmp_path: Path, mutation: _ id="advisory", ), pytest.param(b"name: \xff\n", ["nodes"], ["example-library-worked-example-body"], id="not-utf-8"), + # sdl-yaml/v1 resolves plain scalars with YAML 1.2 Core, so these are strings. + pytest.param( + _VALID_SDL + b"description: 2026-02-30\n", ["nodes", "workflows"], [], id="core-date-spelling-is-a-string" + ), + pytest.param(_VALID_SDL + b"description: =\n", ["nodes", "workflows"], [], id="core-equals-sign-is-a-string"), + pytest.param( + _VALID_SDL + b"description: 2026-02-30\n", + ["stories"], + ["example-library-entry-sections"], + id="core-date-spelling-section-absent", + ), ), ) def test_validated_worked_example_is_parsed_as_sdl( tmp_path: Path, sdl_source: bytes, sdl_sections: list[str], expected: list[str] ) -> None: - repo_root = _seed_repo(tmp_path) - worked_file = repo_root / _WORKED_SDL_PATH - worked_file.parent.mkdir(parents=True) - worked_file.write_bytes(sdl_source) - catalog = _read_catalog(repo_root) - catalog["surfaces"]["scenario"]["worked_examples"][0].update( - path=_WORKED_SDL_PATH, - validation_status="validated", - sdl_sections=sdl_sections, - ) - _write_catalog(repo_root, catalog) + assert _rule_ids(_seed_worked_sdl(tmp_path, sdl_source, {"scenario": sdl_sections})) == expected + + +def test_absent_section_failure_names_the_catalog_entry(tmp_path: Path) -> None: + # Two entries share one file, as the hospital and satcom worked examples do in the catalog. + repo_root = _seed_worked_sdl(tmp_path, _VALID_SDL, {"scenario": ["nodes"], "run": ["stories"]}) + + failures = evaluate_example_library(repo_root) - assert _rule_ids(repo_root) == expected + assert [failure.rule_id for failure in failures] == ["example-library-entry-sections"] + assert "run-worked" in failures[0].message diff --git a/tools/example_library_checks.py b/tools/example_library_checks.py index 894b54038..b5f2e56f7 100644 --- a/tools/example_library_checks.py +++ b/tools/example_library_checks.py @@ -7,7 +7,7 @@ from typing import Any import yaml -from raes import Scenario, parse_sdl, parse_sdl_file +from raes import Scenario, load_sdl_fragment, parse_sdl, parse_sdl_file from tools.policy.common import PolicyFailure from tools.sdl_catalog_parity._paths import _COMPOSITION_FIELDS, _METADATA_FIELDS @@ -91,7 +91,7 @@ def _check_declared_sections( return [ _fail( "example-library-entry-sections", - f"sdl_sections lists sections absent from the validated SDL: {absent}", + f"sdl_sections of entry {entry.get('id')!r} lists sections absent from the validated SDL: {absent}", relative_path, ) ] @@ -133,4 +133,7 @@ def check_worked_example_file(path: Path, relative_path: str, entry: dict[str, A if entry.get("validation_status") != VALIDATED: return [] failures = _check_sdl(path, relative_path, kind="worked-example", subject="worked example") - return failures or _check_declared_sections(entry, yaml.safe_load(path.read_text(encoding="utf-8")), relative_path) + # Use the sdl-yaml/v1 loader that parse_sdl_file used; YAML 1.1 yaml.safe_load can reject valid SDL. + return failures or _check_declared_sections( + entry, load_sdl_fragment(path.read_text(encoding="utf-8")), relative_path + ) From e4907934b34be6485eb06ae25079b527ec319977 Mon Sep 17 00:00:00 2001 From: Yernat Yestekov <2068106+doublewhy@users.noreply.github.com> Date: Mon, 12 Oct 2026 07:43:13 +0000 Subject: [PATCH 03/11] fix(examples): read template files with the sdl-yaml/v1 loader The gate read each template file with yaml.safe_load, which keeps the last of duplicate keys, expands << merge keys, and accepts explicit tags, and then validated a re-dumped copy of the body. A body with a duplicate key, a merge key, or a tag therefore passed the gate while parse_sdl on the same body text rejected it, and an unquoted "description: 2026-02-30" stopped the gate with a ValueError traceback. Template files are now read with load_sdl_fragment, the sdl-yaml/v1 loader that parse_sdl uses. The root mapping holds the template fields, so its keys are read literally, and the body below it gets the same structural key checks as an SDL document root. A file that the loader rejects fails with example-library-parse. Acyclic aliases stay accepted within the loader's limits, as parse_sdl accepts them. The README validation_status row now names the loader and what it rejects. --- examples/README.md | 2 +- ...test_issue_381_example_catalog_metadata.py | 55 +++++++++++++++++++ tools/check_example_library.py | 9 ++- tools/example_library_checks.py | 23 +++++++- 4 files changed, 85 insertions(+), 4 deletions(-) diff --git a/examples/README.md b/examples/README.md index e35012ea7..97cb67b1c 100644 --- a/examples/README.md +++ b/examples/README.md @@ -96,7 +96,7 @@ breaks these rules. | Field | Allowed values | What the gate checks | |-------|----------------|----------------------| -| `validation_status` | `validated` or `guidance` | For `validated`, the gate runs the SDL parser and semantic validator on the worked-example file, or on the template body as PyYAML loads it from the template file, and fails on any error or advisory they report. A `guidance` file is not parsed as SDL. Templates must be `validated`; patterns must be `guidance`. | +| `validation_status` | `validated` or `guidance` | For `validated`, the gate runs the SDL parser and semantic validator on the worked-example file, or on the template body as the `sdl-yaml/v1` loader reads it from the template file, and fails on any error or advisory they report. That loader rejects duplicate keys, merge keys, and explicit tags in the template file. A `guidance` file is not parsed as SDL. Templates must be `validated`; patterns must be `guidance`. | | `sdl_sections` | Top-level section names from [`../specs/sdl/sections.md`](../specs/sdl/sections.md), such as `nodes` or `workflows` | Each name is a current SDL section, not a metadata or composition field such as `name` or `imports`. For a `validated` entry, each listed section is present in the validated SDL. | | `intended_user` | `sdl-author` | The value is in the allowed list. | | `limits` | One or more short statements of what the entry does not show | The list is present and not empty. Reviewers check the wording. | diff --git a/implementations/python/tests/test_issue_381_example_catalog_metadata.py b/implementations/python/tests/test_issue_381_example_catalog_metadata.py index d503d48e2..b8a079627 100644 --- a/implementations/python/tests/test_issue_381_example_catalog_metadata.py +++ b/implementations/python/tests/test_issue_381_example_catalog_metadata.py @@ -145,3 +145,58 @@ def test_absent_section_failure_names_the_catalog_entry(tmp_path: Path) -> None: assert [failure.rule_id for failure in failures] == ["example-library-entry-sections"] assert "run-worked" in failures[0].message + + +_TEMPLATE_PATH = "examples/library/templates/scenario/template.yaml" +_BODY_NAME_LINE = " name: scenario-template\n" + + +@pytest.mark.parametrize( + ("old", "new", "expected", "diagnostic"), + ( + pytest.param( + _BODY_NAME_LINE, + _BODY_NAME_LINE + " description: First.\n description: Second.\n", + ["example-library-parse"], + "sdl.mapping_key_conflict", + id="duplicate-body-key", + ), + pytest.param( + " type: compute\n", + " <<: {type: compute}\n", + ["example-library-parse"], + "sdl.noncanonical_merge", + id="merge-key", + ), + pytest.param( + _BODY_NAME_LINE, + " name: !!str scenario-template\n", + ["example-library-parse"], + "Explicit YAML tags", + id="explicit-tag", + ), + # sdl-yaml/v1 accepts an acyclic alias within its limits, as parse_sdl does. + pytest.param( + _BODY_NAME_LINE, + " name: &shared scenario-template\n description: *shared\n", + [], + "", + id="acyclic-alias-accepted", + ), + # YAML 1.2 Core reads this as a string; yaml.safe_load raises ValueError on it. + pytest.param(_BODY_NAME_LINE, _BODY_NAME_LINE + " description: 2026-02-30\n", [], "", id="core-date-spelling"), + ), +) +def test_template_file_is_read_with_the_sdl_yaml_loader( + tmp_path: Path, old: str, new: str, expected: list[str], diagnostic: str +) -> None: + repo_root = _seed_repo(tmp_path) + template_file = repo_root / _TEMPLATE_PATH + source = template_file.read_text(encoding="utf-8") + assert source.count(old) == 1 + template_file.write_text(source.replace(old, new), encoding="utf-8") + + failures = evaluate_example_library(repo_root) + + assert [failure.rule_id for failure in failures] == expected + assert all(diagnostic in failure.message and failure.path == _TEMPLATE_PATH for failure in failures) diff --git a/tools/check_example_library.py b/tools/check_example_library.py index 54e4dabd0..559d3e9ce 100644 --- a/tools/check_example_library.py +++ b/tools/check_example_library.py @@ -16,7 +16,12 @@ import yaml -from tools.example_library_checks import check_entry_metadata, check_template_body, check_worked_example_file +from tools.example_library_checks import ( + check_entry_metadata, + check_template_body, + check_worked_example_file, + load_template_file, +) from tools.policy.common import PolicyFailure, apply_exceptions, failures_to_json, load_exceptions CATALOG_RELATIVE_PATH = "examples/library/catalog.yaml" @@ -340,7 +345,7 @@ def _check_unique_id(value: Any, owner: str, seen_ids: set[str]) -> list[PolicyF def _check_template_file(path: Path, relative_path: str, *, surface: str, entry: dict[str, Any]) -> list[PolicyFailure]: - raw, failures = _load_yaml_file(path, relative_path) + raw, failures = load_template_file(path, relative_path) if raw is None: return failures catalog_id = entry.get("id") diff --git a/tools/example_library_checks.py b/tools/example_library_checks.py index b5f2e56f7..d6480e37d 100644 --- a/tools/example_library_checks.py +++ b/tools/example_library_checks.py @@ -7,7 +7,7 @@ from typing import Any import yaml -from raes import Scenario, load_sdl_fragment, parse_sdl, parse_sdl_file +from raes import Scenario, SDLParseError, load_sdl_fragment, parse_sdl, parse_sdl_file from tools.policy.common import PolicyFailure from tools.sdl_catalog_parity._paths import _COMPOSITION_FIELDS, _METADATA_FIELDS @@ -116,6 +116,27 @@ def _check_sdl(source: str | Path, relative_path: str, *, kind: str, subject: st return failures +def load_template_file(path: Path, relative_path: str) -> tuple[dict[str, Any] | None, list[PolicyFailure]]: + """Read a template file with the strict sdl-yaml/v1 loader that parse_sdl uses. + + yaml.safe_load keeps the last of duplicate keys and expands merge keys, so + the body it returns can differ from the body an author saves and parses. + The SDL loader rejects both, along with explicit tags. It accepts acyclic + aliases within its limits, as parse_sdl does. The root mapping holds the + template fields, so its keys are read literally; the body below it gets + the same structural key checks as an SDL document root. + """ + try: + raw = load_sdl_fragment(path.read_text(encoding="utf-8"), mapping_keys="literal") + except (SDLParseError, UnicodeDecodeError) as exc: + return None, [_fail("example-library-parse", f"failed to parse {relative_path}: {exc}", relative_path)] + if not isinstance(raw, dict): + return None, [ + _fail("example-library-shape", f"{relative_path} must be a YAML mapping at the top level", relative_path) + ] + return raw, [] + + def check_template_body(entry: dict[str, Any], body: dict[str, Any], relative_path: str) -> list[PolicyFailure]: """Validate a template body as SDL and check the entry's declared sections against it.""" failures = _check_sdl( From b0a87c373cdc9a84aaa9b04212eb73043195b378 Mon Sep 17 00:00:00 2001 From: Yernat Yestekov <2068106+doublewhy@users.noreply.github.com> Date: Mon, 12 Oct 2026 09:17:59 +0000 Subject: [PATCH 04/11] fix(examples): parse template bodies from JSON so loaded strings stay strings check_template_body passed yaml.safe_dump(body) to parse_sdl. PyYAML writes a string such as '0o36' unquoted, and sdl-yaml/v1 reads that back as an int, so the gate accepted a quoted '0o36' interval that parse_sdl rejects in the template file. Serialize the loaded body with json.dumps instead: JSON is sdl-yaml/v1 input and keeps the loaded types. The new quoted-octal-string case fails without this change. --- .../tests/test_issue_381_example_catalog_metadata.py | 8 ++++++++ tools/example_library_checks.py | 8 ++++---- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/implementations/python/tests/test_issue_381_example_catalog_metadata.py b/implementations/python/tests/test_issue_381_example_catalog_metadata.py index b8a079627..b7adb6cf4 100644 --- a/implementations/python/tests/test_issue_381_example_catalog_metadata.py +++ b/implementations/python/tests/test_issue_381_example_catalog_metadata.py @@ -185,6 +185,14 @@ def test_absent_section_failure_names_the_catalog_entry(tmp_path: Path) -> None: ), # YAML 1.2 Core reads this as a string; yaml.safe_load raises ValueError on it. pytest.param(_BODY_NAME_LINE, _BODY_NAME_LINE + " description: 2026-02-30\n", [], "", id="core-date-spelling"), + # sdl-yaml/v1 keeps this quoted value a string; yaml.safe_dump writes it unquoted, which reads back as an int. + pytest.param( + " interval: 30\n", + " interval: '0o36'\n", + ["example-library-template-body"], + "interval must be an integer", + id="quoted-octal-string", + ), ), ) def test_template_file_is_read_with_the_sdl_yaml_loader( diff --git a/tools/example_library_checks.py b/tools/example_library_checks.py index d6480e37d..7381ad200 100644 --- a/tools/example_library_checks.py +++ b/tools/example_library_checks.py @@ -2,11 +2,11 @@ from __future__ import annotations +import json from functools import cache from pathlib import Path from typing import Any -import yaml from raes import Scenario, SDLParseError, load_sdl_fragment, parse_sdl, parse_sdl_file from tools.policy.common import PolicyFailure @@ -139,9 +139,9 @@ def load_template_file(path: Path, relative_path: str) -> tuple[dict[str, Any] | def check_template_body(entry: dict[str, Any], body: dict[str, Any], relative_path: str) -> list[PolicyFailure]: """Validate a template body as SDL and check the entry's declared sections against it.""" - failures = _check_sdl( - yaml.safe_dump(body, sort_keys=False), relative_path, kind="template", subject="template body" - ) + # JSON is sdl-yaml/v1 input that keeps the loaded types; yaml.safe_dump would + # leave a string such as '0o36' unquoted, and sdl-yaml/v1 reads it as an int. + failures = _check_sdl(json.dumps(body), relative_path, kind="template", subject="template body") return failures or _check_declared_sections(entry, body, relative_path) From e44a28b45e035dcc3dd8660ccead69d0543fdd5d Mon Sep 17 00:00:00 2001 From: Yernat Yestekov <2068106+doublewhy@users.noreply.github.com> Date: Mon, 12 Oct 2026 10:37:59 +0000 Subject: [PATCH 05/11] fix(examples): write template bodies as UTF-8 JSON so emoji survive the gate json.dumps(body) escaped a character outside the BMP as a surrogate pair, and the sdl-yaml/v1 loader read the pair back as two lone surrogates, so a template body that parse_sdl accepts, such as `description: smile` plus an emoji, failed the gate with UnicodeEncodeError. The body is now written with ensure_ascii=False. DEL, the C1 controls, U+FFFE and U+FFFF (which the reader rejects) and NEL, U+2028 and U+2029 (which it reads as line breaks) are still written as JSON escapes, so every loaded string reads back unchanged. --- ...test_issue_381_example_catalog_metadata.py | 36 +++++++++++++++++++ tools/example_library_checks.py | 21 +++++++++-- 2 files changed, 54 insertions(+), 3 deletions(-) diff --git a/implementations/python/tests/test_issue_381_example_catalog_metadata.py b/implementations/python/tests/test_issue_381_example_catalog_metadata.py index b7adb6cf4..4764c3a1c 100644 --- a/implementations/python/tests/test_issue_381_example_catalog_metadata.py +++ b/implementations/python/tests/test_issue_381_example_catalog_metadata.py @@ -8,8 +8,10 @@ import pytest import yaml +from raes import load_sdl_fragment from test_example_library_policy import _VALID_BODY, _read_catalog, _seed_repo, _write_catalog from tools.check_example_library import evaluate_example_library +from tools.example_library_checks import sdl_body_source _Mutation = Callable[[dict[str, Any]], None] _WORKED_SDL_PATH = "examples/scenarios/worked.sdl.yaml" @@ -193,6 +195,18 @@ def test_absent_section_failure_names_the_catalog_entry(tmp_path: Path) -> None: "interval must be an integer", id="quoted-octal-string", ), + # parse_sdl accepts this text; a surrogate-pair escape of the emoji would read back as two lone surrogates. + pytest.param( + _BODY_NAME_LINE, _BODY_NAME_LINE + " description: smile \U0001f600\n", [], "", id="non-bmp-character" + ), + # YAML escapes for characters the sdl-yaml/v1 reader rejects or reads as line breaks when written raw. + pytest.param( + _BODY_NAME_LINE, + _BODY_NAME_LINE + ' description: "\\x7f \\x85 \\x9f \\L \\P \\uFFFE \\uFFFF"\n', + [], + "", + id="yaml-special-characters", + ), ), ) def test_template_file_is_read_with_the_sdl_yaml_loader( @@ -208,3 +222,25 @@ def test_template_file_is_read_with_the_sdl_yaml_loader( assert [failure.rule_id for failure in failures] == expected assert all(diagnostic in failure.message and failure.path == _TEMPLATE_PATH for failure in failures) + + +@pytest.mark.parametrize( + "character", + ( + pytest.param("\U0001f600", id="non-bmp"), + pytest.param("\x7f", id="delete"), + pytest.param("\x80", id="first-c1-control"), + pytest.param("\x85", id="next-line"), + pytest.param("\x9f", id="last-c1-control"), + pytest.param("\u2028", id="line-separator"), + pytest.param("\u2029", id="paragraph-separator"), + pytest.param("\ufeff", id="byte-order-mark"), + pytest.param("\ufffe", id="noncharacter-fffe"), + pytest.param("\uffff", id="noncharacter-ffff"), + ), +) +def test_template_body_source_reads_back_as_the_loaded_body(character: str) -> None: + # Spaces around the character catch the line folding that a raw NEL, U+2028, or U+2029 gets. + body = {"name": "scenario-template", "description": f" a {character} b "} + + assert load_sdl_fragment(sdl_body_source(body)) == body diff --git a/tools/example_library_checks.py b/tools/example_library_checks.py index 7381ad200..e0a477968 100644 --- a/tools/example_library_checks.py +++ b/tools/example_library_checks.py @@ -3,6 +3,7 @@ from __future__ import annotations import json +import re from functools import cache from pathlib import Path from typing import Any @@ -25,6 +26,10 @@ # composition rather than as sections. The SDL catalog parity gate keeps these # sets in step with that catalog. NON_SECTION_FIELDS = _METADATA_FIELDS | _COMPOSITION_FIELDS +# Characters that json.dumps(ensure_ascii=False) leaves raw but the sdl-yaml/v1 +# reader rejects (DEL, the C1 controls, U+FFFE, U+FFFF) or reads as a line break +# inside a quoted string (NEL, U+2028, U+2029). JSON escapes the C0 controls. +_YAML_SPECIAL_CHARACTERS = re.compile(r"[\x7f-\x9f\u2028\u2029\ufffe\uffff]") def _fail(rule_id: str, message: str, path: str = CATALOG_RELATIVE_PATH) -> PolicyFailure: @@ -137,11 +142,21 @@ def load_template_file(path: Path, relative_path: str) -> tuple[dict[str, Any] | return raw, [] +def sdl_body_source(body: dict[str, Any]) -> str: + """Write a loaded template body as sdl-yaml/v1 text that reads back to the same values. + + JSON keeps the loaded types; yaml.safe_dump would leave a string such as + '0o36' unquoted, and sdl-yaml/v1 reads it as an int. ensure_ascii=True + would write a character outside the BMP as a surrogate pair, which + sdl-yaml/v1 reads as two lone surrogates. + """ + text = json.dumps(body, ensure_ascii=False) + return _YAML_SPECIAL_CHARACTERS.sub(lambda match: f"\\u{ord(match.group()):04x}", text) + + def check_template_body(entry: dict[str, Any], body: dict[str, Any], relative_path: str) -> list[PolicyFailure]: """Validate a template body as SDL and check the entry's declared sections against it.""" - # JSON is sdl-yaml/v1 input that keeps the loaded types; yaml.safe_dump would - # leave a string such as '0o36' unquoted, and sdl-yaml/v1 reads it as an int. - failures = _check_sdl(json.dumps(body), relative_path, kind="template", subject="template body") + failures = _check_sdl(sdl_body_source(body), relative_path, kind="template", subject="template body") return failures or _check_declared_sections(entry, body, relative_path) From 86a08618e6126f5b79550c1efcbc179e5cf6933e Mon Sep 17 00:00:00 2001 From: Yernat Yestekov <2068106+doublewhy@users.noreply.github.com> Date: Fri, 9 Oct 2026 02:26:50 -0700 Subject: [PATCH 06/11] feat(examples): add scenario templates with validation commands and claim notes Add two scenario templates beside minimal-validated-scenario: segmented-network-scenario (two switched segments, linked hosts with fixed addresses, HTTPS-only access rules, a service feature) and parameterized-scenario (host operating system, size, instance count, and service port from declared variables). Both bodies pass the SDL parser and semantic validator with no advisories, and both are cataloged with the entry metadata. Every template now carries a validation list of commands with their expected results. The gate requires the field, and tools/example_library_checks.py checks that it is a non-empty list of mappings with non-empty command and expected strings. The gate does not run the commands. Each gate entry says that YAML loading keeps the last value of a duplicate key, so the gate does not report duplicate keys. examples/README.md lists what each scenario template shows and does not claim, and how to validate an adapted copy with raes semantic validate from the repository root. Its Template Boundary section defines a ${name} placeholder by the variable-name grammar, says that the gate rejects a placeholder naming an undeclared variable, and says that text such as ${Web OS} is not a placeholder and stays literal text. --- examples/README.md | 50 ++++++++++++-- examples/library/catalog.yaml | 14 ++++ .../action-contract-observation-boundary.yaml | 5 ++ .../templates/run/timed-run-control.yaml | 5 ++ .../scenario/minimal-validated-scenario.yaml | 5 ++ .../scenario/parameterized-scenario.yaml | 62 +++++++++++++++++ .../scenario/segmented-network-scenario.yaml | 69 +++++++++++++++++++ .../study/observational-study-protocol.yaml | 5 ++ .../templates/task/single-objective-task.yaml | 5 ++ .../workflow/parallel-objective-workflow.yaml | 5 ++ .../tests/test_example_library_policy.py | 1 + .../test_issue_378_scenario_templates.py | 44 ++++++++++++ tools/check_example_library.py | 3 + tools/example_library_checks.py | 28 +++++++- 14 files changed, 293 insertions(+), 8 deletions(-) create mode 100644 examples/library/templates/scenario/parameterized-scenario.yaml create mode 100644 examples/library/templates/scenario/segmented-network-scenario.yaml create mode 100644 implementations/python/tests/test_issue_378_scenario_templates.py diff --git a/examples/README.md b/examples/README.md index 97cb67b1c..e19a11981 100644 --- a/examples/README.md +++ b/examples/README.md @@ -73,20 +73,23 @@ The AUT-806 library is indexed by [`library/catalog.yaml`](library/catalog.yaml). It is a versioned, machine-readable catalog for the current non-normative authoring library. -| Surface | Template | Pattern | -|---------|----------|---------| -| Scenario | [`library/templates/scenario/minimal-validated-scenario.yaml`](library/templates/scenario/minimal-validated-scenario.yaml) | [`library/patterns/scenario-reference-integrity.yaml`](library/patterns/scenario-reference-integrity.yaml) | +| Surface | Templates | Pattern | +|---------|-----------|---------| +| Scenario | [`library/templates/scenario/minimal-validated-scenario.yaml`](library/templates/scenario/minimal-validated-scenario.yaml), [`library/templates/scenario/segmented-network-scenario.yaml`](library/templates/scenario/segmented-network-scenario.yaml), [`library/templates/scenario/parameterized-scenario.yaml`](library/templates/scenario/parameterized-scenario.yaml) | [`library/patterns/scenario-reference-integrity.yaml`](library/patterns/scenario-reference-integrity.yaml) | | Workflow | [`library/templates/workflow/parallel-objective-workflow.yaml`](library/templates/workflow/parallel-objective-workflow.yaml) | [`library/patterns/workflow-explicit-control-graph.yaml`](library/patterns/workflow-explicit-control-graph.yaml) | | Participant behavior | [`library/templates/participant_behavior/action-contract-observation-boundary.yaml`](library/templates/participant_behavior/action-contract-observation-boundary.yaml) | [`library/patterns/participant-behavior-contract-binding.yaml`](library/patterns/participant-behavior-contract-binding.yaml) | | Task | [`library/templates/task/single-objective-task.yaml`](library/templates/task/single-objective-task.yaml) | [`library/patterns/task-as-objective-contract.yaml`](library/patterns/task-as-objective-contract.yaml) | | Run | [`library/templates/run/timed-run-control.yaml`](library/templates/run/timed-run-control.yaml) | [`library/patterns/run-window-with-evidence.yaml`](library/patterns/run-window-with-evidence.yaml) | | Study | [`library/templates/study/observational-study-protocol.yaml`](library/templates/study/observational-study-protocol.yaml) | [`library/patterns/observable-study-conditions.yaml`](library/patterns/observable-study-conditions.yaml) | -Each template has metadata plus a complete current-SDL `body`. The +Each template has metadata, a `validation` list of commands with their +expected results, and a complete current-SDL `body`. The `tools/check_example_library.py` policy gate validates catalog shape, stable IDs, referenced paths, AUT-806 requirement references, entry metadata, and every template body and validated worked example through the SDL parser and -semantic validator. +semantic validator. It also checks that each template has a non-empty +`validation` list whose entries have non-empty `command` and `expected` +strings. The gate does not run those commands. ### Catalog entry metadata @@ -108,6 +111,32 @@ not mean that a backend can realize the scenario or that anything has run. A `guidance` entry is reading material, such as a prose pattern or a test module, and the gate does not parse it as SDL. +### Scenario templates + +Each scenario template claims only that current SDL validation accepts its +body with no advisories. + +| Template | Shows | Does not claim | +|----------|-------|----------------| +| [`minimal-validated-scenario`](library/templates/scenario/minimal-validated-scenario.yaml) | One host service, one participant, an objective whose success is an assertion over an observed proposition, and a workflow with one objective step | That a backend can realize the scenario | +| [`segmented-network-scenario`](library/templates/scenario/segmented-network-scenario.yaml) | Two switched segments, a linked host with a fixed address on each, access rules that admit only HTTPS from the user segment into the server segment, and a service feature | That a backend enforces the access rules or installs the feature | +| [`parameterized-scenario`](library/templates/scenario/parameterized-scenario.yaml) | Host operating system, size, instance count, and service port taken from declared variables with defaults and an allowed-values list | That the scenario instantiates with every supplied value; the gate does not instantiate it | + +To check a template as shipped, run the catalog command in +[Validate The Examples](#validate-the-examples). To check an adapted copy, save +its `body` as `my-scenario.sdl.yaml` in the repository root and run this +command from there: + +```shell +uv run --project implementations/python --frozen raes semantic validate my-scenario.sdl.yaml +``` + +The command prints `validate: success` and exits `0` when the SDL parser and +semantic validator accept the file. It does not report semantic-validator +advisories, such as a compute node without `resources`. The `parse_sdl_file` +check in [Validate The Examples](#validate-the-examples) asserts that there +are none. + ## Validate The Examples From the Python implementation directory: @@ -158,11 +187,18 @@ Check the section reference and limitations before adapting a file: ## Template Boundary -There are no placeholder templates in this directory. Files under +No template in this directory contains incomplete SDL. Files under `examples/scenarios/` are positive SDL examples and must load successfully from disk. Files under `examples/library/templates/` are reusable authoring templates with complete SDL bodies; they are validated by the example-library -policy gate. +policy gate. A template body may use `${name}` variable placeholders, as +`parameterized-scenario` does. A placeholder's `name` follows the variable-name +grammar in +[`../specs/sdl/variables-and-instantiation.md`](../specs/sdl/variables-and-instantiation.md) +(lowercase letters, digits, `-`, and `_`), and the gate rejects a body whose +placeholder names a variable that the body does not declare under `variables`. +Text such as `${Web OS}` or `${CUSTOMER_NAME}` is not a placeholder: it stays +literal text, and the gate does not report it as an undeclared variable. Do not add invalid or incomplete SDL files under `examples/scenarios/`. Negative-path examples belong in focused parser, model, validator, contract, or diff --git a/examples/library/catalog.yaml b/examples/library/catalog.yaml index caa9b430f..775cca439 100644 --- a/examples/library/catalog.yaml +++ b/examples/library/catalog.yaml @@ -40,6 +40,20 @@ surfaces: intended_user: sdl-author limits: - Validation covers SDL structure and references; it does not show that a backend can realize the scenario. + - id: segmented-network-scenario + path: examples/library/templates/scenario/segmented-network-scenario.yaml + validation_status: validated + sdl_sections: [nodes, features, infrastructure] + intended_user: sdl-author + limits: + - Validation covers topology structure, addresses, and references; it does not show that a backend enforces the access rules or installs the feature. + - id: parameterized-scenario + path: examples/library/templates/scenario/parameterized-scenario.yaml + validation_status: validated + sdl_sections: [variables, nodes, infrastructure] + intended_user: sdl-author + limits: + - The gate validates the authored body and does not instantiate it with any variable values. patterns: - id: scenario-reference-integrity path: examples/library/patterns/scenario-reference-integrity.yaml diff --git a/examples/library/templates/participant_behavior/action-contract-observation-boundary.yaml b/examples/library/templates/participant_behavior/action-contract-observation-boundary.yaml index 4fff4ac49..0fd5da980 100644 --- a/examples/library/templates/participant_behavior/action-contract-observation-boundary.yaml +++ b/examples/library/templates/participant_behavior/action-contract-observation-boundary.yaml @@ -7,6 +7,11 @@ source_refs: - docs/explain/sdl/validation.md - implementations/python/tests/test_sem_208_participant_behavior.py summary: Participant behavior template binding an agent action to an action contract, observation boundary, and behavior specification. +validation: + - command: uv run --project implementations/python --frozen python tools/check_example_library.py + expected: Exits 0 with no output. The gate loads this file as YAML, parses the loaded body with the SDL parser and semantic validator, and fails on any error or advisory. Loading keeps the last value of a duplicate key, so the gate does not report duplicate keys. + - command: uv run --project implementations/python --frozen raes semantic validate my-scenario.sdl.yaml + expected: 'With the body saved as my-scenario.sdl.yaml, prints "validate: success" and exits 0. This command does not report semantic-validator advisories.' body: name: library-participant-behavior-template description: Participant behavior template with governed action and observation semantics. diff --git a/examples/library/templates/run/timed-run-control.yaml b/examples/library/templates/run/timed-run-control.yaml index 8c3828c1e..ff9d0a490 100644 --- a/examples/library/templates/run/timed-run-control.yaml +++ b/examples/library/templates/run/timed-run-control.yaml @@ -7,6 +7,11 @@ source_refs: - docs/explain/sdl/sections.md - docs/explain/sdl/validation.md summary: Run template binding story/script timing to objective and workflow windows. +validation: + - command: uv run --project implementations/python --frozen python tools/check_example_library.py + expected: Exits 0 with no output. The gate loads this file as YAML, parses the loaded body with the SDL parser and semantic validator, and fails on any error or advisory. Loading keeps the last value of a duplicate key, so the gate does not report duplicate keys. + - command: uv run --project implementations/python --frozen raes semantic validate my-scenario.sdl.yaml + expected: 'With the body saved as my-scenario.sdl.yaml, prints "validate: success" and exits 0. This command does not report semantic-validator advisories.' body: name: library-run-template description: Run-oriented SDL wrapper with a timed story, script, event, objective, and workflow. diff --git a/examples/library/templates/scenario/minimal-validated-scenario.yaml b/examples/library/templates/scenario/minimal-validated-scenario.yaml index 31110cc06..cee66f249 100644 --- a/examples/library/templates/scenario/minimal-validated-scenario.yaml +++ b/examples/library/templates/scenario/minimal-validated-scenario.yaml @@ -7,6 +7,11 @@ source_refs: - docs/explain/sdl/sections.md - docs/explain/sdl/validation.md summary: Minimal current-SDL scenario with a target, participant, objective, and workflow. +validation: + - command: uv run --project implementations/python --frozen python tools/check_example_library.py + expected: Exits 0 with no output. The gate loads this file as YAML, parses the loaded body with the SDL parser and semantic validator, and fails on any error or advisory. Loading keeps the last value of a duplicate key, so the gate does not report duplicate keys. + - command: uv run --project implementations/python --frozen raes semantic validate my-scenario.sdl.yaml + expected: 'With the body saved as my-scenario.sdl.yaml, prints "validate: success" and exits 0. This command does not report semantic-validator advisories.' body: name: library-scenario-template description: Minimal scenario template exercising identity, target, objective, and workflow refs. diff --git a/examples/library/templates/scenario/parameterized-scenario.yaml b/examples/library/templates/scenario/parameterized-scenario.yaml new file mode 100644 index 000000000..d603db49f --- /dev/null +++ b/examples/library/templates/scenario/parameterized-scenario.yaml @@ -0,0 +1,62 @@ +template: raes-library-template +version: 1 +id: parameterized-scenario +surface: scenario +requirement_refs: [AUT-806] +source_refs: + - docs/explain/sdl/sections.md + - docs/explain/sdl/validation.md +summary: Scenario template whose host operating system, size, instance count, and service port come from declared variables. +validation: + - command: uv run --project implementations/python --frozen python tools/check_example_library.py + expected: Exits 0 with no output. The gate loads this file as YAML, parses the loaded body with the SDL parser and semantic validator, which reject a placeholder that names an undeclared variable, and fails on any error or advisory. Loading keeps the last value of a duplicate key, so the gate does not report duplicate keys. + - command: uv run --project implementations/python --frozen raes semantic validate my-scenario.sdl.yaml + expected: 'With the body saved as my-scenario.sdl.yaml, prints "validate: success" and exits 0. This command does not report semantic-validator advisories.' +body: + name: library-parameterized-template + description: Scenario whose host operating system, size, instance count, and service port come from declared variables with defaults. + variables: + web_os: + type: string + default: linux + allowed_values: [linux, windows] + description: Operating system of each web host. + web_ram: + type: string + default: 1 GiB + description: Memory for each web host. + web_cpu: + type: integer + default: 1 + description: CPU count for each web host. + web_count: + type: integer + default: 2 + description: Number of web hosts. + https_port: + type: integer + default: 443 + description: Port of the HTTPS service on each web host. + nodes: + app-net: + type: switch + web: + type: compute + os: ${web_os} + resources: + ram: ${web_ram} + cpu: ${web_cpu} + services: + - port: ${https_port} + protocol: tcp + name: https + infrastructure: + app-net: + count: 1 + properties: + cidr: 10.30.0.0/24 + gateway: 10.30.0.1 + internal: true + web: + count: ${web_count} + links: [app-net] diff --git a/examples/library/templates/scenario/segmented-network-scenario.yaml b/examples/library/templates/scenario/segmented-network-scenario.yaml new file mode 100644 index 000000000..40c5afba1 --- /dev/null +++ b/examples/library/templates/scenario/segmented-network-scenario.yaml @@ -0,0 +1,69 @@ +template: raes-library-template +version: 1 +id: segmented-network-scenario +surface: scenario +requirement_refs: [AUT-806] +source_refs: + - docs/explain/sdl/sections.md + - docs/explain/sdl/validation.md +summary: Two-segment network template with switches, linked hosts, fixed addresses, access rules, and a service feature. +validation: + - command: uv run --project implementations/python --frozen python tools/check_example_library.py + expected: Exits 0 with no output. The gate loads this file as YAML, parses the loaded body with the SDL parser and semantic validator, and fails on any error or advisory. Loading keeps the last value of a duplicate key, so the gate does not report duplicate keys. + - command: uv run --project implementations/python --frozen raes semantic validate my-scenario.sdl.yaml + expected: 'With the body saved as my-scenario.sdl.yaml, prints "validate: success" and exits 0. This command does not report semantic-validator advisories.' +body: + name: library-segmented-network-template + description: Two network segments with one host each, fixed addresses, and access rules that admit only HTTPS from the user segment to the server segment. + nodes: + user-net: + type: switch + server-net: + type: switch + workstation: + type: compute + resources: {ram: 2 GiB, cpu: 1} + web: + type: compute + resources: {ram: 1 GiB, cpu: 1} + features: [web-server] + services: + - {port: 443, protocol: tcp, name: https} + features: + web-server: + type: Service + source: nginx-1.24 + infrastructure: + user-net: + count: 1 + properties: + cidr: 10.20.1.0/24 + gateway: 10.20.1.1 + internal: true + server-net: + count: 1 + properties: + cidr: 10.20.2.0/24 + gateway: 10.20.2.1 + internal: true + acls: + - name: allow-user-https + direction: in + from_net: user-net + protocol: tcp + ports: [443] + action: allow + - name: deny-user-other + direction: in + from_net: user-net + action: deny + workstation: + count: 1 + links: [user-net] + properties: + - user-net: 10.20.1.10 + web: + count: 1 + links: [server-net] + properties: + - server-net: 10.20.2.10 diff --git a/examples/library/templates/study/observational-study-protocol.yaml b/examples/library/templates/study/observational-study-protocol.yaml index 87786bb37..9f2fce134 100644 --- a/examples/library/templates/study/observational-study-protocol.yaml +++ b/examples/library/templates/study/observational-study-protocol.yaml @@ -7,6 +7,11 @@ source_refs: - docs/explain/sdl/sections.md - docs/decisions/adrs/adr-073-scoring-reward-language-scope.md summary: Study template connecting a task outcome to an observable condition, objective, and workflow protocol. Graded scoring lives in the experiment/evaluator plane per ADR-073. +validation: + - command: uv run --project implementations/python --frozen python tools/check_example_library.py + expected: Exits 0 with no output. The gate loads this file as YAML, parses the loaded body with the SDL parser and semantic validator, and fails on any error or advisory. Loading keeps the last value of a duplicate key, so the gate does not report duplicate keys. + - command: uv run --project implementations/python --frozen raes semantic validate my-scenario.sdl.yaml + expected: 'With the body saved as my-scenario.sdl.yaml, prints "validate: success" and exits 0. This command does not report semantic-validator advisories.' body: name: library-study-template description: Study-oriented SDL wrapper whose objective asserts observable success via a condition. Graded scoring and reward are experiment/evaluator-plane concerns per ADR-073, not SDL surfaces. diff --git a/examples/library/templates/task/single-objective-task.yaml b/examples/library/templates/task/single-objective-task.yaml index 5ed8bc93a..a05220978 100644 --- a/examples/library/templates/task/single-objective-task.yaml +++ b/examples/library/templates/task/single-objective-task.yaml @@ -7,6 +7,11 @@ source_refs: - docs/explain/sdl/sections.md - docs/explain/sdl/limitations.md summary: Task template using current SDL objectives and conditions as the materialized task boundary. +validation: + - command: uv run --project implementations/python --frozen python tools/check_example_library.py + expected: Exits 0 with no output. The gate loads this file as YAML, parses the loaded body with the SDL parser and semantic validator, and fails on any error or advisory. Loading keeps the last value of a duplicate key, so the gate does not report duplicate keys. + - command: uv run --project implementations/python --frozen raes semantic validate my-scenario.sdl.yaml + expected: 'With the body saved as my-scenario.sdl.yaml, prints "validate: success" and exits 0. This command does not report semantic-validator advisories.' body: name: library-task-template description: Task-oriented SDL wrapper using a single objective and success condition. diff --git a/examples/library/templates/workflow/parallel-objective-workflow.yaml b/examples/library/templates/workflow/parallel-objective-workflow.yaml index ed8485270..cd38603a6 100644 --- a/examples/library/templates/workflow/parallel-objective-workflow.yaml +++ b/examples/library/templates/workflow/parallel-objective-workflow.yaml @@ -7,6 +7,11 @@ source_refs: - docs/explain/sdl/sections.md - docs/explain/sdl/validation.md summary: Parallel workflow template with explicit branch convergence through a join step. +validation: + - command: uv run --project implementations/python --frozen python tools/check_example_library.py + expected: Exits 0 with no output. The gate loads this file as YAML, parses the loaded body with the SDL parser and semantic validator, and fails on any error or advisory. Loading keeps the last value of a duplicate key, so the gate does not report duplicate keys. + - command: uv run --project implementations/python --frozen raes semantic validate my-scenario.sdl.yaml + expected: 'With the body saved as my-scenario.sdl.yaml, prints "validate: success" and exits 0. This command does not report semantic-validator advisories.' body: name: library-workflow-template description: Workflow template showing parallel objective branches converging at an explicit join. diff --git a/implementations/python/tests/test_example_library_policy.py b/implementations/python/tests/test_example_library_policy.py index ff4c26100..67f256352 100644 --- a/implementations/python/tests/test_example_library_policy.py +++ b/implementations/python/tests/test_example_library_policy.py @@ -161,6 +161,7 @@ def _seed_repo(tmp_path: Path) -> Path: "requirement_refs": [REQUIREMENT_REF], "source_refs": ["docs/explain/sdl/sections.md"], "summary": f"Reusable {surface} template for policy tests.", + "validation": [{"command": "python tools/check_example_library.py", "expected": "exits 0"}], "body": dict(_VALID_BODY, name=f"{surface}-template"), }, ) diff --git a/implementations/python/tests/test_issue_378_scenario_templates.py b/implementations/python/tests/test_issue_378_scenario_templates.py new file mode 100644 index 000000000..bc7813e77 --- /dev/null +++ b/implementations/python/tests/test_issue_378_scenario_templates.py @@ -0,0 +1,44 @@ +"""Template validation-command rules for the example library (issue #378).""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import pytest +import yaml +from test_example_library_policy import _seed_repo, _write_yaml +from tools.check_example_library import evaluate_example_library + +_TEMPLATE_PATH = "examples/library/templates/scenario/template.yaml" +_MISSING = object() + + +@pytest.mark.parametrize( + ("validation", "rule_id"), + ( + pytest.param(_MISSING, "example-library-template-field", id="missing"), + pytest.param([], "example-library-template-validation", id="empty"), + pytest.param("python tools/check_example_library.py", "example-library-template-validation", id="not-a-list"), + pytest.param( + ["python tools/check_example_library.py"], "example-library-template-validation", id="step-not-a-mapping" + ), + pytest.param( + [{"command": "raes semantic validate x.sdl.yaml"}], "example-library-template-validation", id="no-expected" + ), + pytest.param( + [{"command": " ", "expected": "exits 0"}], "example-library-template-validation", id="blank-command" + ), + ), +) +def test_template_validation_violation_is_the_only_failure(tmp_path: Path, validation: Any, rule_id: str) -> None: + repo_root = _seed_repo(tmp_path) + template_path = repo_root / _TEMPLATE_PATH + template = yaml.safe_load(template_path.read_text(encoding="utf-8")) + if validation is _MISSING: + del template["validation"] + else: + template["validation"] = validation + _write_yaml(template_path, template) + + assert [failure.rule_id for failure in evaluate_example_library(repo_root)] == [rule_id] diff --git a/tools/check_example_library.py b/tools/check_example_library.py index 559d3e9ce..0bd51e016 100644 --- a/tools/check_example_library.py +++ b/tools/check_example_library.py @@ -19,6 +19,7 @@ from tools.example_library_checks import ( check_entry_metadata, check_template_body, + check_template_validation, check_worked_example_file, load_template_file, ) @@ -62,6 +63,7 @@ "requirement_refs", "source_refs", "summary", + "validation", "body", ) REQUIRED_PATTERN_FIELDS: tuple[str, ...] = ( @@ -358,6 +360,7 @@ def _check_template_file(path: Path, relative_path: str, *, surface: str, entry: failures.append( _fail("example-library-template-field", f"missing required template field: {field}", relative_path) ) + failures.extend(check_template_validation(raw, relative_path)) body = raw.get("body") if not isinstance(body, dict): diff --git a/tools/example_library_checks.py b/tools/example_library_checks.py index e0a477968..f13e6576c 100644 --- a/tools/example_library_checks.py +++ b/tools/example_library_checks.py @@ -1,4 +1,4 @@ -"""Entry metadata and validated-content checks for the AUT-806 example library gate.""" +"""Entry metadata, template, and worked-example checks for the AUT-806 example library gate.""" from __future__ import annotations @@ -26,6 +26,7 @@ # composition rather than as sections. The SDL catalog parity gate keeps these # sets in step with that catalog. NON_SECTION_FIELDS = _METADATA_FIELDS | _COMPOSITION_FIELDS +VALIDATION_STEP_FIELDS: tuple[str, ...] = ("command", "expected") # Characters that json.dumps(ensure_ascii=False) leaves raw but the sdl-yaml/v1 # reader rejects (DEL, the C1 controls, U+FFFE, U+FFFF) or reads as a line break # inside a quoted string (NEL, U+2028, U+2029). JSON escapes the C0 controls. @@ -173,3 +174,28 @@ def check_worked_example_file(path: Path, relative_path: str, entry: dict[str, A return failures or _check_declared_sections( entry, load_sdl_fragment(path.read_text(encoding="utf-8")), relative_path ) + + +def check_template_validation(raw: dict[str, Any], relative_path: str) -> list[PolicyFailure]: + """Check the shape of a template's validation list; a missing list is a required-field failure.""" + if "validation" not in raw or _is_validation_list(raw["validation"]): + return [] + return [ + _fail( + "example-library-template-validation", + "validation must be a non-empty list of mappings with non-empty command and expected strings", + relative_path, + ) + ] + + +def _is_validation_list(value: object) -> bool: + return ( + isinstance(value, list) + and bool(value) + and all( + isinstance(step, dict) + and all(isinstance(step.get(field), str) and step[field].strip() for field in VALIDATION_STEP_FIELDS) + for step in value + ) + ) From 6c1f1383197d89a556ec313ddf9e60ffea65db9a Mon Sep 17 00:00:00 2001 From: Yernat Yestekov <2068106+doublewhy@users.noreply.github.com> Date: Mon, 12 Oct 2026 07:52:22 +0000 Subject: [PATCH 07/11] fix(examples): say that the template gate rejects duplicate keys The gate step in each template's validation list said that the gate loads the file as YAML, keeps the last value of a duplicate key, and so does not report duplicate keys. The gate now reads template files with the sdl-yaml/v1 loader, which rejects duplicate keys and merge keys, so the eight SDL templates now say that instead. test_issue_378_scenario_templates.py copies the shipped library and, for each template in the catalog, adds a duplicate key to the body and expects only example-library-parse for that template file. With the yaml.safe_load read restored in _check_template_file, all eight cases fail. --- .../action-contract-observation-boundary.yaml | 2 +- .../templates/run/timed-run-control.yaml | 2 +- .../scenario/minimal-validated-scenario.yaml | 2 +- .../scenario/parameterized-scenario.yaml | 2 +- .../scenario/segmented-network-scenario.yaml | 2 +- .../study/observational-study-protocol.yaml | 2 +- .../templates/task/single-objective-task.yaml | 2 +- .../workflow/parallel-objective-workflow.yaml | 2 +- .../test_issue_378_scenario_templates.py | 47 +++++++++++++++++++ 9 files changed, 55 insertions(+), 8 deletions(-) diff --git a/examples/library/templates/participant_behavior/action-contract-observation-boundary.yaml b/examples/library/templates/participant_behavior/action-contract-observation-boundary.yaml index 0fd5da980..f39893a39 100644 --- a/examples/library/templates/participant_behavior/action-contract-observation-boundary.yaml +++ b/examples/library/templates/participant_behavior/action-contract-observation-boundary.yaml @@ -9,7 +9,7 @@ source_refs: summary: Participant behavior template binding an agent action to an action contract, observation boundary, and behavior specification. validation: - command: uv run --project implementations/python --frozen python tools/check_example_library.py - expected: Exits 0 with no output. The gate loads this file as YAML, parses the loaded body with the SDL parser and semantic validator, and fails on any error or advisory. Loading keeps the last value of a duplicate key, so the gate does not report duplicate keys. + expected: Exits 0 with no output. The gate reads this file with the sdl-yaml/v1 loader, which rejects duplicate keys and merge keys. It parses the body with the SDL parser and semantic validator and fails on any error or advisory. - command: uv run --project implementations/python --frozen raes semantic validate my-scenario.sdl.yaml expected: 'With the body saved as my-scenario.sdl.yaml, prints "validate: success" and exits 0. This command does not report semantic-validator advisories.' body: diff --git a/examples/library/templates/run/timed-run-control.yaml b/examples/library/templates/run/timed-run-control.yaml index ff9d0a490..60559f2d4 100644 --- a/examples/library/templates/run/timed-run-control.yaml +++ b/examples/library/templates/run/timed-run-control.yaml @@ -9,7 +9,7 @@ source_refs: summary: Run template binding story/script timing to objective and workflow windows. validation: - command: uv run --project implementations/python --frozen python tools/check_example_library.py - expected: Exits 0 with no output. The gate loads this file as YAML, parses the loaded body with the SDL parser and semantic validator, and fails on any error or advisory. Loading keeps the last value of a duplicate key, so the gate does not report duplicate keys. + expected: Exits 0 with no output. The gate reads this file with the sdl-yaml/v1 loader, which rejects duplicate keys and merge keys. It parses the body with the SDL parser and semantic validator and fails on any error or advisory. - command: uv run --project implementations/python --frozen raes semantic validate my-scenario.sdl.yaml expected: 'With the body saved as my-scenario.sdl.yaml, prints "validate: success" and exits 0. This command does not report semantic-validator advisories.' body: diff --git a/examples/library/templates/scenario/minimal-validated-scenario.yaml b/examples/library/templates/scenario/minimal-validated-scenario.yaml index cee66f249..d32237c0c 100644 --- a/examples/library/templates/scenario/minimal-validated-scenario.yaml +++ b/examples/library/templates/scenario/minimal-validated-scenario.yaml @@ -9,7 +9,7 @@ source_refs: summary: Minimal current-SDL scenario with a target, participant, objective, and workflow. validation: - command: uv run --project implementations/python --frozen python tools/check_example_library.py - expected: Exits 0 with no output. The gate loads this file as YAML, parses the loaded body with the SDL parser and semantic validator, and fails on any error or advisory. Loading keeps the last value of a duplicate key, so the gate does not report duplicate keys. + expected: Exits 0 with no output. The gate reads this file with the sdl-yaml/v1 loader, which rejects duplicate keys and merge keys. It parses the body with the SDL parser and semantic validator and fails on any error or advisory. - command: uv run --project implementations/python --frozen raes semantic validate my-scenario.sdl.yaml expected: 'With the body saved as my-scenario.sdl.yaml, prints "validate: success" and exits 0. This command does not report semantic-validator advisories.' body: diff --git a/examples/library/templates/scenario/parameterized-scenario.yaml b/examples/library/templates/scenario/parameterized-scenario.yaml index d603db49f..2ac5d9ba3 100644 --- a/examples/library/templates/scenario/parameterized-scenario.yaml +++ b/examples/library/templates/scenario/parameterized-scenario.yaml @@ -9,7 +9,7 @@ source_refs: summary: Scenario template whose host operating system, size, instance count, and service port come from declared variables. validation: - command: uv run --project implementations/python --frozen python tools/check_example_library.py - expected: Exits 0 with no output. The gate loads this file as YAML, parses the loaded body with the SDL parser and semantic validator, which reject a placeholder that names an undeclared variable, and fails on any error or advisory. Loading keeps the last value of a duplicate key, so the gate does not report duplicate keys. + expected: Exits 0 with no output. The gate reads this file with the sdl-yaml/v1 loader, which rejects duplicate keys and merge keys. It parses the body with the SDL parser and semantic validator, which reject a placeholder that names an undeclared variable, and fails on any error or advisory. - command: uv run --project implementations/python --frozen raes semantic validate my-scenario.sdl.yaml expected: 'With the body saved as my-scenario.sdl.yaml, prints "validate: success" and exits 0. This command does not report semantic-validator advisories.' body: diff --git a/examples/library/templates/scenario/segmented-network-scenario.yaml b/examples/library/templates/scenario/segmented-network-scenario.yaml index 40c5afba1..9f581799c 100644 --- a/examples/library/templates/scenario/segmented-network-scenario.yaml +++ b/examples/library/templates/scenario/segmented-network-scenario.yaml @@ -9,7 +9,7 @@ source_refs: summary: Two-segment network template with switches, linked hosts, fixed addresses, access rules, and a service feature. validation: - command: uv run --project implementations/python --frozen python tools/check_example_library.py - expected: Exits 0 with no output. The gate loads this file as YAML, parses the loaded body with the SDL parser and semantic validator, and fails on any error or advisory. Loading keeps the last value of a duplicate key, so the gate does not report duplicate keys. + expected: Exits 0 with no output. The gate reads this file with the sdl-yaml/v1 loader, which rejects duplicate keys and merge keys. It parses the body with the SDL parser and semantic validator and fails on any error or advisory. - command: uv run --project implementations/python --frozen raes semantic validate my-scenario.sdl.yaml expected: 'With the body saved as my-scenario.sdl.yaml, prints "validate: success" and exits 0. This command does not report semantic-validator advisories.' body: diff --git a/examples/library/templates/study/observational-study-protocol.yaml b/examples/library/templates/study/observational-study-protocol.yaml index 9f2fce134..b087d7c14 100644 --- a/examples/library/templates/study/observational-study-protocol.yaml +++ b/examples/library/templates/study/observational-study-protocol.yaml @@ -9,7 +9,7 @@ source_refs: summary: Study template connecting a task outcome to an observable condition, objective, and workflow protocol. Graded scoring lives in the experiment/evaluator plane per ADR-073. validation: - command: uv run --project implementations/python --frozen python tools/check_example_library.py - expected: Exits 0 with no output. The gate loads this file as YAML, parses the loaded body with the SDL parser and semantic validator, and fails on any error or advisory. Loading keeps the last value of a duplicate key, so the gate does not report duplicate keys. + expected: Exits 0 with no output. The gate reads this file with the sdl-yaml/v1 loader, which rejects duplicate keys and merge keys. It parses the body with the SDL parser and semantic validator and fails on any error or advisory. - command: uv run --project implementations/python --frozen raes semantic validate my-scenario.sdl.yaml expected: 'With the body saved as my-scenario.sdl.yaml, prints "validate: success" and exits 0. This command does not report semantic-validator advisories.' body: diff --git a/examples/library/templates/task/single-objective-task.yaml b/examples/library/templates/task/single-objective-task.yaml index a05220978..013f9220b 100644 --- a/examples/library/templates/task/single-objective-task.yaml +++ b/examples/library/templates/task/single-objective-task.yaml @@ -9,7 +9,7 @@ source_refs: summary: Task template using current SDL objectives and conditions as the materialized task boundary. validation: - command: uv run --project implementations/python --frozen python tools/check_example_library.py - expected: Exits 0 with no output. The gate loads this file as YAML, parses the loaded body with the SDL parser and semantic validator, and fails on any error or advisory. Loading keeps the last value of a duplicate key, so the gate does not report duplicate keys. + expected: Exits 0 with no output. The gate reads this file with the sdl-yaml/v1 loader, which rejects duplicate keys and merge keys. It parses the body with the SDL parser and semantic validator and fails on any error or advisory. - command: uv run --project implementations/python --frozen raes semantic validate my-scenario.sdl.yaml expected: 'With the body saved as my-scenario.sdl.yaml, prints "validate: success" and exits 0. This command does not report semantic-validator advisories.' body: diff --git a/examples/library/templates/workflow/parallel-objective-workflow.yaml b/examples/library/templates/workflow/parallel-objective-workflow.yaml index cd38603a6..9ceaea0e8 100644 --- a/examples/library/templates/workflow/parallel-objective-workflow.yaml +++ b/examples/library/templates/workflow/parallel-objective-workflow.yaml @@ -9,7 +9,7 @@ source_refs: summary: Parallel workflow template with explicit branch convergence through a join step. validation: - command: uv run --project implementations/python --frozen python tools/check_example_library.py - expected: Exits 0 with no output. The gate loads this file as YAML, parses the loaded body with the SDL parser and semantic validator, and fails on any error or advisory. Loading keeps the last value of a duplicate key, so the gate does not report duplicate keys. + expected: Exits 0 with no output. The gate reads this file with the sdl-yaml/v1 loader, which rejects duplicate keys and merge keys. It parses the body with the SDL parser and semantic validator and fails on any error or advisory. - command: uv run --project implementations/python --frozen raes semantic validate my-scenario.sdl.yaml expected: 'With the body saved as my-scenario.sdl.yaml, prints "validate: success" and exits 0. This command does not report semantic-validator advisories.' body: diff --git a/implementations/python/tests/test_issue_378_scenario_templates.py b/implementations/python/tests/test_issue_378_scenario_templates.py index bc7813e77..666a1b9d0 100644 --- a/implementations/python/tests/test_issue_378_scenario_templates.py +++ b/implementations/python/tests/test_issue_378_scenario_templates.py @@ -2,6 +2,7 @@ from __future__ import annotations +import shutil from pathlib import Path from typing import Any @@ -12,6 +13,18 @@ _TEMPLATE_PATH = "examples/library/templates/scenario/template.yaml" _MISSING = object() +_REPO_ROOT = Path(__file__).resolve().parents[3] +_CATALOG = yaml.safe_load((_REPO_ROOT / "examples/library/catalog.yaml").read_text(encoding="utf-8")) +_CATALOG_PATHS = [ + entry["path"] + for surface in _CATALOG["surfaces"].values() + for field in ("worked_examples", "templates", "patterns") + for entry in surface[field] +] +_SHIPPED_TEMPLATES = sorted( + entry["path"] for surface in _CATALOG["surfaces"].values() for entry in surface["templates"] +) +_BODY_LINE = "\nbody:\n" @pytest.mark.parametrize( @@ -42,3 +55,37 @@ def test_template_validation_violation_is_the_only_failure(tmp_path: Path, valid _write_yaml(template_path, template) assert [failure.rule_id for failure in evaluate_example_library(repo_root)] == [rule_id] + + +def _copy_library(tmp_path: Path) -> Path: + """Copy examples/ and every catalog path outside it, such as the sem-208 test module. + + The copy marks worked examples as guidance, so the gate checks only that + they exist and does not spend most of its time parsing them. + """ + shutil.copytree(_REPO_ROOT / "examples", tmp_path / "examples") + for relative in _CATALOG_PATHS: + if not (tmp_path / relative).exists(): + (tmp_path / relative).parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(_REPO_ROOT / relative, tmp_path / relative) + catalog = yaml.safe_load((tmp_path / "examples/library/catalog.yaml").read_text(encoding="utf-8")) + for surface in catalog["surfaces"].values(): + for entry in surface["worked_examples"]: + entry["validation_status"] = "guidance" + _write_yaml(tmp_path / "examples/library/catalog.yaml", catalog) + return tmp_path + + +@pytest.mark.parametrize("template_path", _SHIPPED_TEMPLATES) +def test_shipped_template_with_a_duplicate_body_key_fails_the_gate(tmp_path: Path, template_path: str) -> None: + # Each template's gate step says that the gate rejects duplicate keys. + repo_root = _copy_library(tmp_path) + template_file = repo_root / template_path + source = template_file.read_text(encoding="utf-8") + assert source.count(_BODY_LINE) == 1 + duplicated = _BODY_LINE + " description: First.\n description: Second.\n" + template_file.write_text(source.replace(_BODY_LINE, duplicated), encoding="utf-8") + + failures = evaluate_example_library(repo_root) + + assert [(failure.rule_id, failure.path) for failure in failures] == [("example-library-parse", template_path)] From 2eb65bb9862b9816a511538309589c561397fa2f Mon Sep 17 00:00:00 2001 From: Yernat Yestekov <2068106+doublewhy@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:10:11 -0700 Subject: [PATCH 08/11] feat(examples): add experiment authoring-input templates for runs and studies Add two templates whose bodies are experiment-authoring-input-v1 documents: seeded-run-plan (run count, seed, and episode controls for one task) and two-condition-study-design (a treatment factor, two compared conditions, and the runs per condition). Both pass the experiment authoring-input loader, parse_experiment_spec. A template's catalog entry may now declare contract: experiment-authoring-input-v1. Such an entry names experiment-author as its intended user and lists no sdl_sections, and the gate validates its body with the experiment loader instead of the SDL parser. Worked examples and patterns stay SDL. examples/README.md separates the SDL, experiment-input, guidance, and unsupported parts of the task, run, and study surfaces, including why there is no experiment-task-v1 template, and gives a command to validate an adapted experiment template. --- docs/explain/getting-started.md | 2 +- examples/README.md | 103 +++++++++++---- examples/library/catalog.yaml | 23 +++- .../templates/run/seeded-run-plan.yaml | 38 ++++++ .../study/two-condition-study-design.yaml | 65 ++++++++++ .../test_issue_380_experiment_templates.py | 117 ++++++++++++++++++ tools/example_library_checks.py | 81 +++++++++--- 7 files changed, 381 insertions(+), 48 deletions(-) create mode 100644 examples/library/templates/run/seeded-run-plan.yaml create mode 100644 examples/library/templates/study/two-condition-study-design.yaml create mode 100644 implementations/python/tests/test_issue_380_experiment_templates.py diff --git a/docs/explain/getting-started.md b/docs/explain/getting-started.md index 56c4ac352..566f5f452 100644 --- a/docs/explain/getting-started.md +++ b/docs/explain/getting-started.md @@ -77,7 +77,7 @@ Use the lowest level that answers the question. | Orientation | You need to know what RAES is and is not. | README, docs index, reference map | Current repository scope and entrypoints | SDL validity, backend behavior, or experiment adequacy | | SDL parse and validation | You have an SDL file and need current parser feedback. | `parse_sdl_file()`, `load_scenario()`, SDL parser/model/validator tests | Structural and semantic acceptance by the reference implementation | Deployment viability or general domain completeness | | Example-backed authoring | You need a worked scenario to study or adapt. | `examples/scenarios/*.sdl.yaml`, `test_example_schema_conformance.py` | The example loads from disk without advisories under current tests | Suitability for another range, backend, exercise, or research design | -| Template and pattern authoring | You need a reusable starting shape for a scenario, workflow, participant behavior, task, run, or study. | `examples/library/catalog.yaml`, `tools/check_example_library.py` | The cataloged template body validates as current SDL and the pattern has stable metadata | New runtime semantics or first-class task, run, or study sections | +| Template and pattern authoring | You need a reusable starting shape for a scenario, workflow, participant behavior, task, run, or study. | `examples/library/catalog.yaml`, `tools/check_example_library.py` | The cataloged template body validates as current SDL or as an `experiment-authoring-input-v1` document, and the pattern has stable metadata | New runtime semantics or first-class task, run, or study sections | | Runtime and contracts | You need processor or backend integration context. | Runtime compiler/planner, contract schemas, backend profiles, conformance fixtures | Current reference-stack and contract behavior | Production backend correctness or operational reliability | | Specification review | You need to evaluate a semantic or authority claim. | `specs/`, ADRs, formal notes, tests | The current reasoning and normative boundary for a claim | Completed implementation when the materialized code/contracts are absent | diff --git a/examples/README.md b/examples/README.md index e19a11981..1e6d1a0fd 100644 --- a/examples/README.md +++ b/examples/README.md @@ -79,37 +79,44 @@ machine-readable catalog for the current non-normative authoring library. | Workflow | [`library/templates/workflow/parallel-objective-workflow.yaml`](library/templates/workflow/parallel-objective-workflow.yaml) | [`library/patterns/workflow-explicit-control-graph.yaml`](library/patterns/workflow-explicit-control-graph.yaml) | | Participant behavior | [`library/templates/participant_behavior/action-contract-observation-boundary.yaml`](library/templates/participant_behavior/action-contract-observation-boundary.yaml) | [`library/patterns/participant-behavior-contract-binding.yaml`](library/patterns/participant-behavior-contract-binding.yaml) | | Task | [`library/templates/task/single-objective-task.yaml`](library/templates/task/single-objective-task.yaml) | [`library/patterns/task-as-objective-contract.yaml`](library/patterns/task-as-objective-contract.yaml) | -| Run | [`library/templates/run/timed-run-control.yaml`](library/templates/run/timed-run-control.yaml) | [`library/patterns/run-window-with-evidence.yaml`](library/patterns/run-window-with-evidence.yaml) | -| Study | [`library/templates/study/observational-study-protocol.yaml`](library/templates/study/observational-study-protocol.yaml) | [`library/patterns/observable-study-conditions.yaml`](library/patterns/observable-study-conditions.yaml) | +| Run | [`library/templates/run/timed-run-control.yaml`](library/templates/run/timed-run-control.yaml), [`library/templates/run/seeded-run-plan.yaml`](library/templates/run/seeded-run-plan.yaml) | [`library/patterns/run-window-with-evidence.yaml`](library/patterns/run-window-with-evidence.yaml) | +| Study | [`library/templates/study/observational-study-protocol.yaml`](library/templates/study/observational-study-protocol.yaml), [`library/templates/study/two-condition-study-design.yaml`](library/templates/study/two-condition-study-design.yaml) | [`library/patterns/observable-study-conditions.yaml`](library/patterns/observable-study-conditions.yaml) | Each template has metadata, a `validation` list of commands with their -expected results, and a complete current-SDL `body`. The +expected results, and a complete `body`. Most bodies are current SDL; the +`seeded-run-plan` and `two-condition-study-design` bodies are +`experiment-authoring-input-v1` documents. The `tools/check_example_library.py` policy gate validates catalog shape, stable -IDs, referenced paths, AUT-806 requirement references, entry metadata, and -every template body and validated worked example through the SDL parser and -semantic validator. It also checks that each template has a non-empty -`validation` list whose entries have non-empty `command` and `expected` -strings. The gate does not run those commands. +IDs, referenced paths, AUT-806 requirement references, and entry metadata. It +checks every SDL template body and validated worked example with the SDL +parser and semantic validator, and every experiment template body with the +experiment authoring-input loader. It also checks that each template has a +non-empty `validation` list whose entries have non-empty `command` and +`expected` strings. The gate does not run those commands. ### Catalog entry metadata Every `worked_examples`, `templates`, and `patterns` entry in the catalog -records four fields. The policy gate rejects an entry that omits a field or -breaks these rules. +records `validation_status`, `intended_user`, and `limits`. SDL entries also +record `sdl_sections`, and a template whose body is not SDL records +`contract`. The policy gate rejects an entry that omits a field or breaks +these rules. | Field | Allowed values | What the gate checks | |-------|----------------|----------------------| -| `validation_status` | `validated` or `guidance` | For `validated`, the gate runs the SDL parser and semantic validator on the worked-example file, or on the template body as the `sdl-yaml/v1` loader reads it from the template file, and fails on any error or advisory they report. That loader rejects duplicate keys, merge keys, and explicit tags in the template file. A `guidance` file is not parsed as SDL. Templates must be `validated`; patterns must be `guidance`. | -| `sdl_sections` | Top-level section names from [`../specs/sdl/sections.md`](../specs/sdl/sections.md), such as `nodes` or `workflows` | Each name is a current SDL section, not a metadata or composition field such as `name` or `imports`. For a `validated` entry, each listed section is present in the validated SDL. | -| `intended_user` | `sdl-author` | The value is in the allowed list. | +| `contract` | `sdl-yaml/v1`, the default when absent, or `experiment-authoring-input-v1` for a template | The value is allowed for the entry. It selects how a `validated` body is checked and which `intended_user` the entry names. | +| `validation_status` | `validated` or `guidance` | For `validated`, the gate checks the template body or worked-example file against its contract. SDL goes through the SDL parser and semantic validator, on the worked-example file or on the template body as the `sdl-yaml/v1` loader reads it from the template file, and fails on any error or advisory they report. That loader rejects duplicate keys, merge keys, and explicit tags in the template file. `experiment-authoring-input-v1` goes through the experiment authoring-input loader and fails on any error. A `guidance` file is not checked against a contract. Templates must be `validated`; patterns must be `guidance`. | +| `sdl_sections` | Top-level section names from [`../specs/sdl/sections.md`](../specs/sdl/sections.md), such as `nodes` or `workflows` | Present for SDL entries and absent otherwise. Each name is a current SDL section, not a metadata or composition field such as `name` or `imports`. For a `validated` entry, each listed section is present in the validated SDL. | +| `intended_user` | `sdl-author` for SDL entries, `experiment-author` for `experiment-authoring-input-v1` templates | The value matches the entry's contract. | | `limits` | One or more short statements of what the entry does not show | The list is present and not empty. Reviewers check the wording. | `sdl_sections` names the sections that an entry shows for its surface, so a large worked example lists only some of the sections it uses. `validated` -means that current SDL validation accepts the file or template body. It does -not mean that a backend can realize the scenario or that anything has run. A -`guidance` entry is reading material, such as a prose pattern or a test -module, and the gate does not parse it as SDL. +means that the contract's validator accepts the file or template body. It does +not mean that a backend can realize the scenario, that anything has run, or +that references to other artifacts resolve. A `guidance` entry is reading +material, such as a prose pattern or a test module, and the gate does not +check it against a contract. ### Scenario templates @@ -137,6 +144,46 @@ advisories, such as a compute node without `resources`. The `parse_sdl_file` check in [Validate The Examples](#validate-the-examples) asserts that there are none. +### Task, run, and study entries + +The task, run, and study surfaces keep these parts separate: + +- **SDL:** the `single-objective-task`, `timed-run-control`, and + `observational-study-protocol` templates and the worked examples. SDL has no + `tasks`, `runs`, or `studies` sections, so these use objectives, workflows, + timing, and observable conditions. +- **Experiment authoring input:** the `seeded-run-plan` and + `two-condition-study-design` templates. Each body is an + `experiment-authoring-input-v1` document, the pre-run design that binds a + task to a run plan + ([ADR-074](../docs/decisions/adrs/adr-074-experiment-authoring-input-contract-boundary.md)). + `seeded-run-plan` declares the run count, seed, and episode controls for one + task. `two-condition-study-design` declares a treatment factor, two compared + conditions, and the runs per condition. +- **Guidance:** the task, run, and study patterns, which are prose. +- **Outside current support:** + - An experiment task template. An experiment task is an + `experiment-task-v1` document that a run plan names in `task_ref`. The + repository has no task loader, `raes semantic validate` does not accept + the `experiment-task-v1` contract, and every task needs an artifact + reference with a checksum, size, and creation time that a reusable + template would have to invent. + - Resolving `task_ref` and checking condition factor levels against the + declared factors. The experiment authoring-input loader does neither. + - Admitting, scheduling, or running a plan, and the `experiment-run-v1` and + `experiment-study-v1` records that describe actual executions and + analyses. + +To check an adapted copy of an experiment template, save its `body` as +`my-experiment.exp.yaml` and run: + +```shell +uv run --project implementations/python --frozen python -c "import sys; from pathlib import Path; from raes_contracts.experiment_spec import load_experiment_spec; print(load_experiment_spec(Path(sys.argv[1])).spec_id)" my-experiment.exp.yaml +``` + +The command prints the document's `spec_id` and exits `0`. An invalid +document raises `ExperimentSpecValidationError` and exits `1`. + ## Validate The Examples From the Python implementation directory: @@ -190,10 +237,10 @@ Check the section reference and limitations before adapting a file: No template in this directory contains incomplete SDL. Files under `examples/scenarios/` are positive SDL examples and must load successfully from disk. Files under `examples/library/templates/` are reusable authoring -templates with complete SDL bodies; they are validated by the example-library -policy gate. A template body may use `${name}` variable placeholders, as -`parameterized-scenario` does. A placeholder's `name` follows the variable-name -grammar in +templates with complete SDL or `experiment-authoring-input-v1` bodies; they are +validated by the example-library policy gate. An SDL template body may use +`${name}` variable placeholders, as `parameterized-scenario` does. A +placeholder's `name` follows the variable-name grammar in [`../specs/sdl/variables-and-instantiation.md`](../specs/sdl/variables-and-instantiation.md) (lowercase letters, digits, `-`, and `_`), and the gate rejects a body whose placeholder names a variable that the body does not declare under `variables`. @@ -204,9 +251,11 @@ Do not add invalid or incomplete SDL files under `examples/scenarios/`. Negative-path examples belong in focused parser, model, validator, contract, or conformance tests. -Task, run, and study templates use current SDL wrappers rather than first-class -`tasks`, `runs`, or `studies` sections. They show how to express those concepts -with objectives, workflows, timing, observable conditions, and evidence-like -references that the current implementation can validate. Graded scoring and -reward are experiment/evaluator-plane concerns (experiment-* contracts) per -ADR-073, not SDL surfaces. +The task, run, and study SDL templates use current SDL wrappers rather than +first-class `tasks`, `runs`, or `studies` sections. They show how to express +those concepts with objectives, workflows, timing, observable conditions, and +evidence-like references that the current implementation can validate. Graded +scoring and reward are experiment/evaluator-plane concerns (experiment-* +contracts) per ADR-073, not SDL surfaces. The run and study experiment +templates are pre-run designs; see +[Task, run, and study entries](#task-run-and-study-entries). diff --git a/examples/library/catalog.yaml b/examples/library/catalog.yaml index 775cca439..7d49a13da 100644 --- a/examples/library/catalog.yaml +++ b/examples/library/catalog.yaml @@ -9,9 +9,10 @@ description: > Domain-neutral RAES agentic-environment authoring library for non-normative examples, templates, and reusable patterns. Current examples are strongest in cyber; additional domains extend through their own profiles, - assets, backends, and evidence requirements. Template bodies use current SDL - so they can be parsed and semantically validated by the Python reference - implementation. + assets, backends, and evidence requirements. SDL template bodies are parsed + and semantically validated by the Python reference implementation, and + experiment templates hold experiment-authoring-input-v1 documents that its + experiment authoring-input loader validates. surfaces: scenario: summary: Complete scenario authoring from topology through objective flow. @@ -162,6 +163,14 @@ surfaces: intended_user: sdl-author limits: - SDL has no first-class runs section; the template expresses a run with stories, scripts, and events, and nothing is scheduled or run. + - id: seeded-run-plan + path: examples/library/templates/run/seeded-run-plan.yaml + contract: experiment-authoring-input-v1 + validation_status: validated + intended_user: experiment-author + limits: + - Pre-run design only; nothing runs, and it is not an experiment-run-v1 record. + - The loader does not resolve task_ref, and the seed is not an executable random-stream contract. patterns: - id: run-window-with-evidence path: examples/library/patterns/run-window-with-evidence.yaml @@ -189,6 +198,14 @@ surfaces: intended_user: sdl-author limits: - SDL has no first-class studies section; graded scoring stays in the experiment/evaluator plane per ADR-073. + - id: two-condition-study-design + path: examples/library/templates/study/two-condition-study-design.yaml + contract: experiment-authoring-input-v1 + validation_status: validated + intended_user: experiment-author + limits: + - Pre-run design only; it is not an experiment-study-v1 record and declares no analysis plan. + - The loader does not resolve task_ref or check condition factor levels against the declared factors. patterns: - id: observable-study-conditions path: examples/library/patterns/observable-study-conditions.yaml diff --git a/examples/library/templates/run/seeded-run-plan.yaml b/examples/library/templates/run/seeded-run-plan.yaml new file mode 100644 index 000000000..ea903a04f --- /dev/null +++ b/examples/library/templates/run/seeded-run-plan.yaml @@ -0,0 +1,38 @@ +template: raes-library-template +version: 1 +id: seeded-run-plan +surface: run +requirement_refs: [AUT-806, EXP-736] +source_refs: + - docs/decisions/adrs/adr-074-experiment-authoring-input-contract-boundary.md + - examples/experiments/techvault-smoke-run.exp.yaml +summary: Pre-run experiment specification for repeated, seeded runs of one task with sequential turns and a step limit. +validation: + - command: uv run --project implementations/python --frozen python tools/check_example_library.py + expected: Exits 0 with no output. The gate validates this body with the experiment authoring-input loader, parse_experiment_spec, and fails on any error. + - command: 'uv run --project implementations/python --frozen python -c "import sys; from pathlib import Path; from raes_contracts.experiment_spec import load_experiment_spec; print(load_experiment_spec(Path(sys.argv[1])).spec_id)" my-experiment.exp.yaml' + expected: With the body saved as my-experiment.exp.yaml, prints its spec_id and exits 0. An invalid document raises ExperimentSpecValidationError and exits 1. +body: + schema_version: experiment-authoring-input/v1 + spec_id: library-seeded-run-plan + spec_version: 1.0.0 + title: Seeded run plan + description: Ten seeded runs of one task with sequential turns and a fixed step limit. + task_ref: + ref_kind: task + ref_id: task-example-v1 + ref_version: 1.0.0 + run_plan: + stochastic_controls: + - control_id: base-seed + role: seed + value: 1234 + description: Base seed for the plan. How each run derives its seed is not declared here. + episode_control: + turn_order: sequential + max_steps: 50 + termination_rule: End each episode after 50 logical steps. + target_run_count: 10 + validity_notes: + - category: reproducibility + note: The seed is descriptive. It does not make this plan an executable random-stream contract. diff --git a/examples/library/templates/study/two-condition-study-design.yaml b/examples/library/templates/study/two-condition-study-design.yaml new file mode 100644 index 000000000..b85f3bb4c --- /dev/null +++ b/examples/library/templates/study/two-condition-study-design.yaml @@ -0,0 +1,65 @@ +template: raes-library-template +version: 1 +id: two-condition-study-design +surface: study +requirement_refs: [AUT-806, EXP-736] +source_refs: + - docs/decisions/adrs/adr-074-experiment-authoring-input-contract-boundary.md + - examples/experiments/techvault-red-tactic-sweep.exp.yaml +summary: Pre-run experiment specification comparing two levels of one treatment factor with a fixed number of runs per condition. +validation: + - command: uv run --project implementations/python --frozen python tools/check_example_library.py + expected: Exits 0 with no output. The gate validates this body with the experiment authoring-input loader, parse_experiment_spec, and fails on any error. + - command: 'uv run --project implementations/python --frozen python -c "import sys; from pathlib import Path; from raes_contracts.experiment_spec import load_experiment_spec; print(load_experiment_spec(Path(sys.argv[1])).spec_id)" my-experiment.exp.yaml' + expected: With the body saved as my-experiment.exp.yaml, prints its spec_id and exits 0. An invalid document raises ExperimentSpecValidationError and exits 1. +body: + schema_version: experiment-authoring-input/v1 + spec_id: library-two-condition-study-design + spec_version: 1.0.0 + title: Two-condition study design + description: Compares two levels of one treatment factor with the same number of seeded runs per condition. + task_ref: + ref_kind: task + ref_id: task-example-v1 + ref_version: 1.0.0 + factors: + guidance-level: + name: Guidance level + factor_kind: treatment + levels: [baseline, with-hints] + run_plan: + stochastic_controls: + - control_id: base-seed + role: seed + value: 1234 + description: Base seed shared by both conditions. + episode_control: + turn_order: sequential + max_steps: 50 + termination_rule: End each episode after 50 logical steps. + allocation: + allocation_unit: run + allocation_method: balanced + compared_conditions: [cond-baseline, cond-with-hints] + condition_assignments: + cond-baseline: + condition_id: cond-baseline + factor_levels: + guidance-level: baseline + required_parameters: + - name: guidance_level + value: baseline + value_kind: protocol + cond-with-hints: + condition_id: cond-with-hints + factor_levels: + guidance-level: with-hints + required_parameters: + - name: guidance_level + value: with-hints + value_kind: protocol + target_runs_per_condition: 20 + replication_policy: Twenty independent seeded runs per condition. + validity_notes: + - category: statistical + note: The design fixes the runs per condition before any run. It names no analysis plan, estimand, or uncertainty procedure. diff --git a/implementations/python/tests/test_issue_380_experiment_templates.py b/implementations/python/tests/test_issue_380_experiment_templates.py new file mode 100644 index 000000000..13845fdd3 --- /dev/null +++ b/implementations/python/tests/test_issue_380_experiment_templates.py @@ -0,0 +1,117 @@ +"""Experiment authoring-input templates in the example library (issue #380).""" + +from __future__ import annotations + +import copy +from collections.abc import Callable +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import pytest +import yaml +from test_example_library_policy import _VALID_BODY, _read_catalog, _seed_repo, _write_catalog, _write_yaml +from tools.check_example_library import evaluate_example_library + +_TEMPLATE_PATH = "examples/library/templates/run/template.yaml" +_EXPERIMENT_CONTRACT = "experiment-authoring-input-v1" +_EXPERIMENT_BODY: dict[str, Any] = { + "schema_version": "experiment-authoring-input/v1", + "spec_id": "policy-test-spec", + "spec_version": "1.0.0", + "title": "Policy test spec", + "description": "Experiment template body for policy tests.", + "task_ref": {"ref_kind": "task", "ref_id": "task-policy-test"}, + "run_plan": { + "episode_control": {"turn_order": "sequential", "termination_rule": "End each episode after one step."}, + "target_run_count": 1, + }, +} + + +@dataclass(frozen=True) +class _Library: + """The seeded catalog and the run surface's template file, loaded for editing.""" + + catalog: dict[str, Any] + template: dict[str, Any] + + def entry(self, surface: str = "run", field: str = "templates") -> dict[str, Any]: + return self.catalog["surfaces"][surface][field][0] + + +_Mutation = Callable[[_Library], None] + + +def _apply(repo_root: Path, mutate: _Mutation) -> None: + template_path = repo_root / _TEMPLATE_PATH + library = _Library(_read_catalog(repo_root), yaml.safe_load(template_path.read_text(encoding="utf-8"))) + mutate(library) + _write_catalog(repo_root, library.catalog) + _write_yaml(template_path, library.template) + + +def _as_experiment_template(library: _Library) -> None: + entry = library.entry() + entry.update(contract=_EXPERIMENT_CONTRACT, intended_user="experiment-author") + del entry["sdl_sections"] + library.template["body"] = copy.deepcopy(_EXPERIMENT_BODY) + + +def _seed_experiment_repo(tmp_path: Path) -> Path: + repo_root = _seed_repo(tmp_path) + _apply(repo_root, _as_experiment_template) + return repo_root + + +def _set_entry(key: str, value: Any, surface: str = "run", field: str = "templates") -> _Mutation: + def mutate(library: _Library) -> None: + library.entry(surface, field)[key] = value + + return mutate + + +def _drop_task_ref(library: _Library) -> None: + del library.template["body"]["task_ref"] + + +def _sdl_body(library: _Library) -> None: + library.template["body"] = copy.deepcopy(_VALID_BODY) + + +def _drop_contract(library: _Library) -> None: + entry = library.entry() + del entry["contract"] + entry.update(intended_user="sdl-author", sdl_sections=["objectives"]) + + +def test_experiment_template_passes(tmp_path: Path) -> None: + assert evaluate_example_library(_seed_experiment_repo(tmp_path)) == [] + + +@pytest.mark.parametrize( + ("mutation", "rule_id"), + ( + pytest.param( + _set_entry("contract", _EXPERIMENT_CONTRACT, field="worked_examples"), + "example-library-entry-contract", + id="contract-on-worked-example", + ), + pytest.param(_set_entry("contract", "experiment-run-v1"), "example-library-entry-contract", id="unknown"), + pytest.param(_set_entry("sdl_sections", ["objectives"]), "example-library-entry-sections", id="sections"), + pytest.param(_set_entry("intended_user", "sdl-author"), "example-library-entry-user", id="sdl-author"), + pytest.param( + _set_entry("intended_user", "experiment-author", surface="scenario"), + "example-library-entry-user", + id="experiment-author-on-sdl", + ), + pytest.param(_drop_task_ref, "example-library-template-body", id="invalid-experiment-body"), + pytest.param(_sdl_body, "example-library-template-body", id="sdl-body-under-experiment-contract"), + pytest.param(_drop_contract, "example-library-template-body", id="experiment-body-without-contract"), + ), +) +def test_experiment_template_violation_is_the_only_failure(tmp_path: Path, mutation: _Mutation, rule_id: str) -> None: + repo_root = _seed_experiment_repo(tmp_path) + _apply(repo_root, mutation) + + assert [failure.rule_id for failure in evaluate_example_library(repo_root)] == [rule_id] diff --git a/tools/example_library_checks.py b/tools/example_library_checks.py index f13e6576c..d9b789474 100644 --- a/tools/example_library_checks.py +++ b/tools/example_library_checks.py @@ -8,7 +8,9 @@ from pathlib import Path from typing import Any +import yaml from raes import Scenario, SDLParseError, load_sdl_fragment, parse_sdl, parse_sdl_file +from raes_contracts.experiment_spec import ExperimentSpecValidationError, parse_experiment_spec from tools.policy.common import PolicyFailure from tools.sdl_catalog_parity._paths import _COMPOSITION_FIELDS, _METADATA_FIELDS @@ -21,7 +23,14 @@ "templates": VALIDATED, "patterns": GUIDANCE, } -INTENDED_USERS: tuple[str, ...] = ("sdl-author",) +SDL_CONTRACT = "sdl-yaml/v1" +EXPERIMENT_CONTRACT = "experiment-authoring-input-v1" +# Each body contract has exactly one intended user. Only templates may use a +# contract other than SDL; an entry without a contract is SDL. +CONTRACT_USERS: dict[str, str] = { + SDL_CONTRACT: "sdl-author", + EXPERIMENT_CONTRACT: "experiment-author", +} # Top-level SDL fields that specs/sdl/sections.md classifies as metadata or # composition rather than as sections. The SDL catalog parity gate keeps these # sets in step with that catalog. @@ -49,7 +58,7 @@ def _sdl_section_names() -> frozenset[str]: def check_entry_metadata(field: str, entry: dict[str, Any], owner: str) -> list[PolicyFailure]: - """Check the validation status, SDL sections, intended user, and limits of one catalog entry.""" + """Check the validation status, limits, contract, intended user, and SDL sections of one catalog entry.""" failures: list[PolicyFailure] = [] allowed_statuses = ( (REQUIRED_VALIDATION_STATUS[field],) if field in REQUIRED_VALIDATION_STATUS else VALIDATION_STATUSES @@ -62,31 +71,44 @@ def check_entry_metadata(field: str, entry: dict[str, Any], owner: str) -> list[ f"{owner}.validation_status must be one of {list(allowed_statuses)}; got {status!r}", ) ) - user = entry.get("intended_user") - if user not in INTENDED_USERS: - failures.append( - _fail( - "example-library-entry-user", - f"{owner}.intended_user must be one of {list(INTENDED_USERS)}; got {user!r}", - ) - ) if not _text_list(entry.get("limits")): failures.append(_fail("example-library-entry-limits", f"{owner}.limits must be a non-empty list of strings")) - sections = _text_list(entry.get("sdl_sections")) - if not sections or len(set(sections)) != len(sections): + contract = entry.get("contract", SDL_CONTRACT) + allowed_contracts = tuple(CONTRACT_USERS) if field == "templates" else (SDL_CONTRACT,) + if contract not in allowed_contracts: failures.append( - _fail("example-library-entry-sections", f"{owner}.sdl_sections must be a non-empty list of distinct names") + _fail( + "example-library-entry-contract", + f"{owner}.contract must be one of {list(allowed_contracts)}; got {contract!r}", + ) ) - elif unknown := sorted(set(sections) - _sdl_section_names()): + return failures + user = entry.get("intended_user") + if user != CONTRACT_USERS[contract]: failures.append( _fail( - "example-library-entry-sections", - f"{owner}.sdl_sections names values that are not SDL sections: {unknown}", + "example-library-entry-user", + f"{owner}.intended_user must be {CONTRACT_USERS[contract]!r} for {contract}; got {user!r}", ) ) + failures.extend(_check_entry_sections(entry, owner, contract)) return failures +def _check_entry_sections(entry: dict[str, Any], owner: str, contract: str) -> list[PolicyFailure]: + if contract != SDL_CONTRACT: + absent_failure = _fail("example-library-entry-sections", f"{owner}.sdl_sections must be absent for {contract}") + return [absent_failure] if "sdl_sections" in entry else [] + sections = _text_list(entry.get("sdl_sections")) + if not sections or len(set(sections)) != len(sections): + return [ + _fail("example-library-entry-sections", f"{owner}.sdl_sections must be a non-empty list of distinct names") + ] + unknown = sorted(set(sections) - _sdl_section_names()) + message = f"{owner}.sdl_sections names values that are not SDL sections: {unknown}" + return [_fail("example-library-entry-sections", message)] if unknown else [] + + def _check_declared_sections( entry: dict[str, Any], document: dict[str, Any], relative_path: str ) -> list[PolicyFailure]: @@ -155,8 +177,33 @@ def sdl_body_source(body: dict[str, Any]) -> str: return _YAML_SPECIAL_CHARACTERS.sub(lambda match: f"\\u{ord(match.group()):04x}", text) +def _check_experiment_spec(body: dict[str, Any], relative_path: str) -> list[PolicyFailure]: + """Validate a template body with the experiment authoring-input loader (ADR-074).""" + try: + parse_experiment_spec(yaml.safe_dump(body, sort_keys=False)) + except ExperimentSpecValidationError as exc: + return [ + _fail( + "example-library-template-body", + f"template body is not a valid {EXPERIMENT_CONTRACT} document: {exc}", + relative_path, + ) + ] + return [] + + def check_template_body(entry: dict[str, Any], body: dict[str, Any], relative_path: str) -> list[PolicyFailure]: - """Validate a template body as SDL and check the entry's declared sections against it.""" + """Validate a template body against the entry's contract. + + An SDL body must also contain the entry's declared sections. A body under + an unknown contract is not validated; the entry metadata check reports + the contract. + """ + contract = entry.get("contract", SDL_CONTRACT) + if contract == EXPERIMENT_CONTRACT: + return _check_experiment_spec(body, relative_path) + if contract != SDL_CONTRACT: + return [] failures = _check_sdl(sdl_body_source(body), relative_path, kind="template", subject="template body") return failures or _check_declared_sections(entry, body, relative_path) From c21fc786ca35dd6655a28c34de97110142013f7a Mon Sep 17 00:00:00 2001 From: Yernat Yestekov <2068106+doublewhy@users.noreply.github.com> Date: Sat, 10 Oct 2026 10:11:47 -0700 Subject: [PATCH 09/11] fix(examples): match experiment template claims to the gate and loader The gate reads a template file with yaml.safe_load, so duplicate keys, scalar aliases, and << merge keys never reach parse_experiment_spec, while the template's own loader command rejects them in a saved copy. The README validation_status row and both templates' gate `expected` text now say so. The README and catalog said the loader never checks condition factor levels. It does check the levels that a binding descriptor (under binding_semantics: explicit-required, ADR-094) or a stratified selection stratum joins. Only unjoined levels, such as those in the shipped two-condition-study-design body, go unchecked. The gate now allows the experiment-authoring-input-v1 contract only on run and study templates. The run and study surface summaries mention the experiment templates, the checker docstring and getting-started.md name both validation paths, and the adapted-copy command says to run it from the repository root. --- docs/explain/getting-started.md | 2 +- examples/README.md | 17 +++++++--- examples/library/catalog.yaml | 7 ++-- .../templates/run/seeded-run-plan.yaml | 2 +- .../study/two-condition-study-design.yaml | 2 +- .../test_issue_380_experiment_templates.py | 34 +++++++++++++++---- tools/check_example_library.py | 6 ++-- tools/example_library_checks.py | 8 +++-- 8 files changed, 55 insertions(+), 23 deletions(-) diff --git a/docs/explain/getting-started.md b/docs/explain/getting-started.md index 566f5f452..a62da8fe3 100644 --- a/docs/explain/getting-started.md +++ b/docs/explain/getting-started.md @@ -60,7 +60,7 @@ fixtures or schema authority. | Understand the repository | `README.md`, [`docs/index.md`](../index.md), [`docs/explain/reference/canonical-reference-map.md`](reference/canonical-reference-map.md) | Read the referenced docs | The repository layout and current boundaries are understood. | | Understand the agentic-environment lifecycle | [`docs/explain/reference/glossary.md`](reference/glossary.md), [`docs/explain/sdl/runtime-architecture.md`](sdl/runtime-architecture.md) | Follow the authored scenario, realized environment, evidence, and conformance references | RAES system concepts are distinguished from SDL and backend behavior. | | Read a complete scenario | `examples/README.md`, `examples/scenarios/*.sdl.yaml` | `pytest tests/test_example_schema_conformance.py` | The checked examples load through the current parser boundary. | -| Start from a reusable template or pattern | `examples/library/catalog.yaml`, `examples/library/templates/`, `examples/library/patterns/` | `python tools/check_example_library.py` | The catalog covers scenario, workflow, participant behavior, task, run, and study surfaces with parser-validated template bodies. | +| Start from a reusable template or pattern | `examples/library/catalog.yaml`, `examples/library/templates/`, `examples/library/patterns/` | `python tools/check_example_library.py` | The catalog covers scenario, workflow, participant behavior, task, run, and study surfaces, with template bodies validated as SDL or as `experiment-authoring-input-v1` documents. | | Author a small SDL file | [`docs/explain/sdl/index.md`](sdl/index.md), [`docs/explain/sdl/sections.md`](sdl/sections.md), [`docs/explain/sdl/validation.md`](sdl/validation.md) | `parse_sdl_file()` or `load_scenario()` | The file is accepted by the current SDL model and semantic validator. | | Use an agent-facing authoring surface | `raes-mcp`, then `raes_tool_surface`, `raes_agent_guidance`, `raes_intended_use_profiles`, and [`docs/explain/sdl/language-service.md`](sdl/language-service.md) | `raes_agent_guidance`, `raes_intended_use_profiles`, `sdl_completions`, `sdl_apply_edit`, `sdl_diagnostics`, `sdl_format`, `sdl_references`, `sdl_validate`, `sdl_design_assessment`, `sdl_plan`, `sdl_claims_assessment` | The agent can choose an intended-use scope, inspect current RAES blockers, and help author, edit, dry-run, and qualify claims without repository-local code access. | | Use variables or imports | [`docs/explain/sdl/parser.md`](sdl/parser.md), [`docs/explain/sdl/sections.md`](sdl/sections.md) | `raes sdl resolve`, `raes sdl verify-imports` | Imports and variable placeholders follow the current parser rules. | diff --git a/examples/README.md b/examples/README.md index 1e6d1a0fd..f866b6bf0 100644 --- a/examples/README.md +++ b/examples/README.md @@ -104,8 +104,8 @@ these rules. | Field | Allowed values | What the gate checks | |-------|----------------|----------------------| -| `contract` | `sdl-yaml/v1`, the default when absent, or `experiment-authoring-input-v1` for a template | The value is allowed for the entry. It selects how a `validated` body is checked and which `intended_user` the entry names. | -| `validation_status` | `validated` or `guidance` | For `validated`, the gate checks the template body or worked-example file against its contract. SDL goes through the SDL parser and semantic validator, on the worked-example file or on the template body as the `sdl-yaml/v1` loader reads it from the template file, and fails on any error or advisory they report. That loader rejects duplicate keys, merge keys, and explicit tags in the template file. `experiment-authoring-input-v1` goes through the experiment authoring-input loader and fails on any error. A `guidance` file is not checked against a contract. Templates must be `validated`; patterns must be `guidance`. | +| `contract` | `sdl-yaml/v1`, the default when absent, or `experiment-authoring-input-v1` for a run or study template | The value is allowed for the entry. It selects how a `validated` body is checked and which `intended_user` the entry names. | +| `validation_status` | `validated` or `guidance` | For `validated`, the gate checks the template body or worked-example file against its contract. SDL goes through the SDL parser and semantic validator, on the worked-example file or on the template body as the `sdl-yaml/v1` loader reads it from the template file, and fails on any error or advisory they report. That loader rejects duplicate keys, merge keys, and explicit tags in the template file. `experiment-authoring-input-v1` goes through the experiment authoring-input loader and fails on any error it reports for the body as PyYAML `safe_load` reads it from the template file. That read keeps the last of duplicate keys and expands scalar aliases and `<<` merge keys, which the loader rejects in a saved copy. A `guidance` file is not checked against a contract. Templates must be `validated`; patterns must be `guidance`. | | `sdl_sections` | Top-level section names from [`../specs/sdl/sections.md`](../specs/sdl/sections.md), such as `nodes` or `workflows` | Present for SDL entries and absent otherwise. Each name is a current SDL section, not a metadata or composition field such as `name` or `imports`. For a `validated` entry, each listed section is present in the validated SDL. | | `intended_user` | `sdl-author` for SDL entries, `experiment-author` for `experiment-authoring-input-v1` templates | The value matches the entry's contract. | | `limits` | One or more short statements of what the entry does not show | The list is present and not empty. Reviewers check the wording. | @@ -168,14 +168,21 @@ The task, run, and study surfaces keep these parts separate: the `experiment-task-v1` contract, and every task needs an artifact reference with a checksum, size, and creation time that a reusable template would have to invent. - - Resolving `task_ref` and checking condition factor levels against the - declared factors. The experiment authoring-input loader does neither. + - Resolving `task_ref`, and checking condition factor levels that no + binding descriptor or stratified selection stratum joins. The experiment + authoring-input loader does neither. It does check joined levels against + the declared factors and the condition assignment: binding descriptors + under `binding_semantics: explicit-required` + ([ADR-094](../docs/decisions/adrs/adr-094-authoritative-cross-plane-experiment-bindings.md)) + and stratified selection strata. The shipped `two-condition-study-design` + body joins none, so the loader does not check its levels. - Admitting, scheduling, or running a plan, and the `experiment-run-v1` and `experiment-study-v1` records that describe actual executions and analyses. To check an adapted copy of an experiment template, save its `body` as -`my-experiment.exp.yaml` and run: +`my-experiment.exp.yaml` in the repository root and run this command from +there: ```shell uv run --project implementations/python --frozen python -c "import sys; from pathlib import Path; from raes_contracts.experiment_spec import load_experiment_spec; print(load_experiment_spec(Path(sys.argv[1])).spec_id)" my-experiment.exp.yaml diff --git a/examples/library/catalog.yaml b/examples/library/catalog.yaml index 7d49a13da..a86690c74 100644 --- a/examples/library/catalog.yaml +++ b/examples/library/catalog.yaml @@ -145,7 +145,7 @@ surfaces: limits: - SDL has no first-class tasks section, and an objective written this way is not an experiment-task-v1 record. run: - summary: Run-oriented authoring with story, script, event, workflow, and objective windows. + summary: Run-oriented authoring with story, script, event, workflow, and objective windows, plus a pre-run experiment run plan. worked_examples: - id: hospital-run-story path: examples/scenarios/hospital-ransomware-surgery-day.sdl.yaml @@ -180,7 +180,7 @@ surfaces: limits: - SDL has no first-class runs section; the pattern maps a run onto stories, scripts, events, and objective windows. study: - summary: Study-oriented authoring with observable conditions, objectives, and workflow protocol. Graded scoring is an experiment/evaluator-plane concern per ADR-073. + summary: Study-oriented authoring with observable conditions, objectives, and workflow protocol, plus a pre-run two-condition experiment design. Graded scoring is an experiment/evaluator-plane concern per ADR-073. worked_examples: - id: satcom-study-scenario path: examples/scenarios/satcom-release-poisoning.sdl.yaml @@ -205,7 +205,8 @@ surfaces: intended_user: experiment-author limits: - Pre-run design only; it is not an experiment-study-v1 record and declares no analysis plan. - - The loader does not resolve task_ref or check condition factor levels against the declared factors. + - The loader does not resolve task_ref. As shipped, the body joins no factor level through an explicit-required binding descriptor or a stratified selection stratum, so the loader does not check its condition factor levels against the declared factors. + - The required_parameters are descriptive data that bind no configuration target; binding_semantics explicit-required rejects them and needs binding descriptors instead (ADR-094). patterns: - id: observable-study-conditions path: examples/library/patterns/observable-study-conditions.yaml diff --git a/examples/library/templates/run/seeded-run-plan.yaml b/examples/library/templates/run/seeded-run-plan.yaml index ea903a04f..a7ba0a9ed 100644 --- a/examples/library/templates/run/seeded-run-plan.yaml +++ b/examples/library/templates/run/seeded-run-plan.yaml @@ -9,7 +9,7 @@ source_refs: summary: Pre-run experiment specification for repeated, seeded runs of one task with sequential turns and a step limit. validation: - command: uv run --project implementations/python --frozen python tools/check_example_library.py - expected: Exits 0 with no output. The gate validates this body with the experiment authoring-input loader, parse_experiment_spec, and fails on any error. + expected: Exits 0 with no output. The gate passes this body, as yaml.safe_load reads it, to parse_experiment_spec and fails on any error that it reports. It does not see duplicate keys, scalar aliases, or << merge keys; the second command rejects them. - command: 'uv run --project implementations/python --frozen python -c "import sys; from pathlib import Path; from raes_contracts.experiment_spec import load_experiment_spec; print(load_experiment_spec(Path(sys.argv[1])).spec_id)" my-experiment.exp.yaml' expected: With the body saved as my-experiment.exp.yaml, prints its spec_id and exits 0. An invalid document raises ExperimentSpecValidationError and exits 1. body: diff --git a/examples/library/templates/study/two-condition-study-design.yaml b/examples/library/templates/study/two-condition-study-design.yaml index b85f3bb4c..94224aede 100644 --- a/examples/library/templates/study/two-condition-study-design.yaml +++ b/examples/library/templates/study/two-condition-study-design.yaml @@ -9,7 +9,7 @@ source_refs: summary: Pre-run experiment specification comparing two levels of one treatment factor with a fixed number of runs per condition. validation: - command: uv run --project implementations/python --frozen python tools/check_example_library.py - expected: Exits 0 with no output. The gate validates this body with the experiment authoring-input loader, parse_experiment_spec, and fails on any error. + expected: Exits 0 with no output. The gate passes this body, as yaml.safe_load reads it, to parse_experiment_spec and fails on any error that it reports. It does not see duplicate keys, scalar aliases, or << merge keys; the second command rejects them. - command: 'uv run --project implementations/python --frozen python -c "import sys; from pathlib import Path; from raes_contracts.experiment_spec import load_experiment_spec; print(load_experiment_spec(Path(sys.argv[1])).spec_id)" my-experiment.exp.yaml' expected: With the body saved as my-experiment.exp.yaml, prints its spec_id and exits 0. An invalid document raises ExperimentSpecValidationError and exits 1. body: diff --git a/implementations/python/tests/test_issue_380_experiment_templates.py b/implementations/python/tests/test_issue_380_experiment_templates.py index 13845fdd3..385b28038 100644 --- a/implementations/python/tests/test_issue_380_experiment_templates.py +++ b/implementations/python/tests/test_issue_380_experiment_templates.py @@ -13,7 +13,6 @@ from test_example_library_policy import _VALID_BODY, _read_catalog, _seed_repo, _write_catalog, _write_yaml from tools.check_example_library import evaluate_example_library -_TEMPLATE_PATH = "examples/library/templates/run/template.yaml" _EXPERIMENT_CONTRACT = "experiment-authoring-input-v1" _EXPERIMENT_BODY: dict[str, Any] = { "schema_version": "experiment-authoring-input/v1", @@ -31,21 +30,23 @@ @dataclass(frozen=True) class _Library: - """The seeded catalog and the run surface's template file, loaded for editing.""" + """The seeded catalog and one surface's template file, loaded for editing.""" catalog: dict[str, Any] template: dict[str, Any] + surface: str = "run" - def entry(self, surface: str = "run", field: str = "templates") -> dict[str, Any]: - return self.catalog["surfaces"][surface][field][0] + def entry(self, surface: str | None = None, field: str = "templates") -> dict[str, Any]: + return self.catalog["surfaces"][surface or self.surface][field][0] _Mutation = Callable[[_Library], None] -def _apply(repo_root: Path, mutate: _Mutation) -> None: - template_path = repo_root / _TEMPLATE_PATH - library = _Library(_read_catalog(repo_root), yaml.safe_load(template_path.read_text(encoding="utf-8"))) +def _apply(repo_root: Path, mutate: _Mutation, surface: str = "run") -> None: + template_path = repo_root / f"examples/library/templates/{surface}/template.yaml" + template = yaml.safe_load(template_path.read_text(encoding="utf-8")) + library = _Library(_read_catalog(repo_root), template, surface) mutate(library) _write_catalog(repo_root, library.catalog) _write_yaml(template_path, library.template) @@ -115,3 +116,22 @@ def test_experiment_template_violation_is_the_only_failure(tmp_path: Path, mutat _apply(repo_root, mutation) assert [failure.rule_id for failure in evaluate_example_library(repo_root)] == [rule_id] + + +@pytest.mark.parametrize( + ("surface", "rule_ids"), + ( + pytest.param("study", [], id="study"), + *( + pytest.param(surface, ["example-library-entry-contract"], id=surface) + for surface in ("scenario", "workflow", "participant_behavior", "task") + ), + ), +) +def test_experiment_contract_is_limited_to_run_and_study_templates( + tmp_path: Path, surface: str, rule_ids: list[str] +) -> None: + repo_root = _seed_repo(tmp_path) + _apply(repo_root, _as_experiment_template, surface=surface) + + assert [failure.rule_id for failure in evaluate_example_library(repo_root)] == rule_ids diff --git a/tools/check_example_library.py b/tools/check_example_library.py index 0bd51e016..ed18e07e2 100644 --- a/tools/check_example_library.py +++ b/tools/check_example_library.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # ruff: noqa: E402, I001 -"""Structural and SDL-validation gate for the AUT-806 example library.""" +"""Structural, SDL, and experiment authoring-input validation gate for the AUT-806 example library.""" from __future__ import annotations @@ -264,7 +264,7 @@ def _check_reference_entry( ) failures.extend(_check_unique_id(entry.get("id"), f"surfaces.{surface}.{field}[{index}]", seen_ids)) - failures.extend(check_entry_metadata(field, entry, f"surfaces.{surface}.{field}[{index}]")) + failures.extend(check_entry_metadata(field, entry, f"surfaces.{surface}.{field}[{index}]", surface=surface)) absolute, relative, path_failure = _repo_relative_path( entry.get("path"), repo_root=repo_root, @@ -314,7 +314,7 @@ def _check_artifact_entries( ) entry_id = entry.get("id") failures.extend(_check_unique_id(entry_id, f"surfaces.{surface}.{field}[{index}]", seen_ids)) - failures.extend(check_entry_metadata(field, entry, f"surfaces.{surface}.{field}[{index}]")) + failures.extend(check_entry_metadata(field, entry, f"surfaces.{surface}.{field}[{index}]", surface=surface)) absolute, relative, path_failure = _repo_relative_path( entry.get("path"), repo_root=repo_root, diff --git a/tools/example_library_checks.py b/tools/example_library_checks.py index d9b789474..b0de72707 100644 --- a/tools/example_library_checks.py +++ b/tools/example_library_checks.py @@ -31,6 +31,9 @@ SDL_CONTRACT: "sdl-author", EXPERIMENT_CONTRACT: "experiment-author", } +# An authoring input is a pre-run design that names its task by reference +# (ADR-074), so only run and study templates may hold one. +EXPERIMENT_SURFACES: frozenset[str] = frozenset({"run", "study"}) # Top-level SDL fields that specs/sdl/sections.md classifies as metadata or # composition rather than as sections. The SDL catalog parity gate keeps these # sets in step with that catalog. @@ -57,7 +60,7 @@ def _sdl_section_names() -> frozenset[str]: return frozenset(Scenario.model_fields) - NON_SECTION_FIELDS -def check_entry_metadata(field: str, entry: dict[str, Any], owner: str) -> list[PolicyFailure]: +def check_entry_metadata(field: str, entry: dict[str, Any], owner: str, *, surface: str) -> list[PolicyFailure]: """Check the validation status, limits, contract, intended user, and SDL sections of one catalog entry.""" failures: list[PolicyFailure] = [] allowed_statuses = ( @@ -74,7 +77,8 @@ def check_entry_metadata(field: str, entry: dict[str, Any], owner: str) -> list[ if not _text_list(entry.get("limits")): failures.append(_fail("example-library-entry-limits", f"{owner}.limits must be a non-empty list of strings")) contract = entry.get("contract", SDL_CONTRACT) - allowed_contracts = tuple(CONTRACT_USERS) if field == "templates" else (SDL_CONTRACT,) + experiment_allowed = field == "templates" and surface in EXPERIMENT_SURFACES + allowed_contracts = tuple(CONTRACT_USERS) if experiment_allowed else (SDL_CONTRACT,) if contract not in allowed_contracts: failures.append( _fail( From 4f14577d667e75b92aa3846085fc207b61d7289b Mon Sep 17 00:00:00 2001 From: Yernat Yestekov <2068106+doublewhy@users.noreply.github.com> Date: Mon, 12 Oct 2026 08:03:22 +0000 Subject: [PATCH 10/11] fix(examples): read experiment templates with the experiment loader's reader The gate read every template file with yaml.safe_load, so an experiment template body with a duplicate key, a scalar alias, or a << merge key passed the gate while load_experiment_spec rejected the same body saved on its own. The README and both experiment templates said so. The template loader now follows the entry's contract. An experiment-authoring-input-v1 template file is read with _load_yaml, the bounded reader that parse_experiment_spec uses, which rejects duplicate keys, aliases, and merge keys. Any other template keeps the sdl-yaml/v1 loader, which would reject the hyphenated factor and condition ids of two-condition-study-design as non-canonical SDL field spellings. A file that its reader rejects fails with example-library-parse. The README validation_status row and the gate step of seeded-run-plan and two-condition-study-design now say that the gate rejects these. --- examples/README.md | 2 +- .../templates/run/seeded-run-plan.yaml | 2 +- .../study/two-condition-study-design.yaml | 2 +- .../test_issue_380_experiment_templates.py | 26 +++++++++++++++++++ tools/check_example_library.py | 2 +- tools/example_library_checks.py | 26 +++++++++++++------ 6 files changed, 48 insertions(+), 12 deletions(-) diff --git a/examples/README.md b/examples/README.md index f866b6bf0..c9cf53256 100644 --- a/examples/README.md +++ b/examples/README.md @@ -105,7 +105,7 @@ these rules. | Field | Allowed values | What the gate checks | |-------|----------------|----------------------| | `contract` | `sdl-yaml/v1`, the default when absent, or `experiment-authoring-input-v1` for a run or study template | The value is allowed for the entry. It selects how a `validated` body is checked and which `intended_user` the entry names. | -| `validation_status` | `validated` or `guidance` | For `validated`, the gate checks the template body or worked-example file against its contract. SDL goes through the SDL parser and semantic validator, on the worked-example file or on the template body as the `sdl-yaml/v1` loader reads it from the template file, and fails on any error or advisory they report. That loader rejects duplicate keys, merge keys, and explicit tags in the template file. `experiment-authoring-input-v1` goes through the experiment authoring-input loader and fails on any error it reports for the body as PyYAML `safe_load` reads it from the template file. That read keeps the last of duplicate keys and expands scalar aliases and `<<` merge keys, which the loader rejects in a saved copy. A `guidance` file is not checked against a contract. Templates must be `validated`; patterns must be `guidance`. | +| `validation_status` | `validated` or `guidance` | For `validated`, the gate checks the template body or worked-example file against its contract. SDL goes through the SDL parser and semantic validator, on the worked-example file or on the template body as the `sdl-yaml/v1` loader reads it from the template file, and fails on any error or advisory they report. That loader rejects duplicate keys, merge keys, and explicit tags in the template file. `experiment-authoring-input-v1` goes through the experiment authoring-input loader, on the template body as that loader's YAML reader reads it from the template file, and fails on any error it reports. That reader rejects duplicate keys, aliases, and `<<` merge keys in the template file. A `guidance` file is not checked against a contract. Templates must be `validated`; patterns must be `guidance`. | | `sdl_sections` | Top-level section names from [`../specs/sdl/sections.md`](../specs/sdl/sections.md), such as `nodes` or `workflows` | Present for SDL entries and absent otherwise. Each name is a current SDL section, not a metadata or composition field such as `name` or `imports`. For a `validated` entry, each listed section is present in the validated SDL. | | `intended_user` | `sdl-author` for SDL entries, `experiment-author` for `experiment-authoring-input-v1` templates | The value matches the entry's contract. | | `limits` | One or more short statements of what the entry does not show | The list is present and not empty. Reviewers check the wording. | diff --git a/examples/library/templates/run/seeded-run-plan.yaml b/examples/library/templates/run/seeded-run-plan.yaml index a7ba0a9ed..3c84bc1f7 100644 --- a/examples/library/templates/run/seeded-run-plan.yaml +++ b/examples/library/templates/run/seeded-run-plan.yaml @@ -9,7 +9,7 @@ source_refs: summary: Pre-run experiment specification for repeated, seeded runs of one task with sequential turns and a step limit. validation: - command: uv run --project implementations/python --frozen python tools/check_example_library.py - expected: Exits 0 with no output. The gate passes this body, as yaml.safe_load reads it, to parse_experiment_spec and fails on any error that it reports. It does not see duplicate keys, scalar aliases, or << merge keys; the second command rejects them. + expected: Exits 0 with no output. The gate reads this file with the YAML reader of the experiment authoring-input loader, which rejects duplicate keys, aliases, and << merge keys. It passes the body to parse_experiment_spec and fails on any error that it reports. - command: 'uv run --project implementations/python --frozen python -c "import sys; from pathlib import Path; from raes_contracts.experiment_spec import load_experiment_spec; print(load_experiment_spec(Path(sys.argv[1])).spec_id)" my-experiment.exp.yaml' expected: With the body saved as my-experiment.exp.yaml, prints its spec_id and exits 0. An invalid document raises ExperimentSpecValidationError and exits 1. body: diff --git a/examples/library/templates/study/two-condition-study-design.yaml b/examples/library/templates/study/two-condition-study-design.yaml index 94224aede..0c346b125 100644 --- a/examples/library/templates/study/two-condition-study-design.yaml +++ b/examples/library/templates/study/two-condition-study-design.yaml @@ -9,7 +9,7 @@ source_refs: summary: Pre-run experiment specification comparing two levels of one treatment factor with a fixed number of runs per condition. validation: - command: uv run --project implementations/python --frozen python tools/check_example_library.py - expected: Exits 0 with no output. The gate passes this body, as yaml.safe_load reads it, to parse_experiment_spec and fails on any error that it reports. It does not see duplicate keys, scalar aliases, or << merge keys; the second command rejects them. + expected: Exits 0 with no output. The gate reads this file with the YAML reader of the experiment authoring-input loader, which rejects duplicate keys, aliases, and << merge keys. It passes the body to parse_experiment_spec and fails on any error that it reports. - command: 'uv run --project implementations/python --frozen python -c "import sys; from pathlib import Path; from raes_contracts.experiment_spec import load_experiment_spec; print(load_experiment_spec(Path(sys.argv[1])).spec_id)" my-experiment.exp.yaml' expected: With the body saved as my-experiment.exp.yaml, prints its spec_id and exits 0. An invalid document raises ExperimentSpecValidationError and exits 1. body: diff --git a/implementations/python/tests/test_issue_380_experiment_templates.py b/implementations/python/tests/test_issue_380_experiment_templates.py index 385b28038..8d474f5d9 100644 --- a/implementations/python/tests/test_issue_380_experiment_templates.py +++ b/implementations/python/tests/test_issue_380_experiment_templates.py @@ -135,3 +135,29 @@ def test_experiment_contract_is_limited_to_run_and_study_templates( _apply(repo_root, _as_experiment_template, surface=surface) assert [failure.rule_id for failure in evaluate_example_library(repo_root)] == rule_ids + + +_RUN_TEMPLATE_PATH = "examples/library/templates/run/template.yaml" +_TITLE_LINE = " title: Policy test spec\n" +_TEXT_LINES = _TITLE_LINE + " description: Experiment template body for policy tests.\n" + + +@pytest.mark.parametrize( + ("old", "new"), + ( + pytest.param(_TITLE_LINE, _TITLE_LINE + " title: Second title\n", id="duplicate-key"), + pytest.param(_TEXT_LINES, " title: &text Policy test spec\n description: *text\n", id="scalar-alias"), + pytest.param(" turn_order: sequential\n", " <<: {turn_order: sequential}\n", id="merge-key"), + ), +) +def test_experiment_template_file_is_read_with_the_experiment_loader_reader(tmp_path: Path, old: str, new: str) -> None: + # yaml.safe_load accepts each edit, and the body it returns is a valid experiment spec. + repo_root = _seed_experiment_repo(tmp_path) + template_file = repo_root / _RUN_TEMPLATE_PATH + source = template_file.read_text(encoding="utf-8") + assert source.count(old) == 1 + template_file.write_text(source.replace(old, new), encoding="utf-8") + + failures = evaluate_example_library(repo_root) + + assert [(failure.rule_id, failure.path) for failure in failures] == [("example-library-parse", _RUN_TEMPLATE_PATH)] diff --git a/tools/check_example_library.py b/tools/check_example_library.py index ed18e07e2..ec7a152e1 100644 --- a/tools/check_example_library.py +++ b/tools/check_example_library.py @@ -347,7 +347,7 @@ def _check_unique_id(value: Any, owner: str, seen_ids: set[str]) -> list[PolicyF def _check_template_file(path: Path, relative_path: str, *, surface: str, entry: dict[str, Any]) -> list[PolicyFailure]: - raw, failures = load_template_file(path, relative_path) + raw, failures = load_template_file(path, relative_path, entry.get("contract")) if raw is None: return failures catalog_id = entry.get("id") diff --git a/tools/example_library_checks.py b/tools/example_library_checks.py index b0de72707..af2fe5ab9 100644 --- a/tools/example_library_checks.py +++ b/tools/example_library_checks.py @@ -11,6 +11,7 @@ import yaml from raes import Scenario, SDLParseError, load_sdl_fragment, parse_sdl, parse_sdl_file from raes_contracts.experiment_spec import ExperimentSpecValidationError, parse_experiment_spec +from raes_contracts.experiment_spec import _load_yaml as _read_experiment_yaml from tools.policy.common import PolicyFailure from tools.sdl_catalog_parity._paths import _COMPOSITION_FIELDS, _METADATA_FIELDS @@ -148,19 +149,28 @@ def _check_sdl(source: str | Path, relative_path: str, *, kind: str, subject: st return failures -def load_template_file(path: Path, relative_path: str) -> tuple[dict[str, Any] | None, list[PolicyFailure]]: - """Read a template file with the strict sdl-yaml/v1 loader that parse_sdl uses. +def load_template_file( + path: Path, relative_path: str, contract: object = None +) -> tuple[dict[str, Any] | None, list[PolicyFailure]]: + """Read a template file with the strict loader of the entry's contract. yaml.safe_load keeps the last of duplicate keys and expands merge keys, so the body it returns can differ from the body an author saves and parses. - The SDL loader rejects both, along with explicit tags. It accepts acyclic - aliases within its limits, as parse_sdl does. The root mapping holds the - template fields, so its keys are read literally; the body below it gets - the same structural key checks as an SDL document root. + An experiment authoring-input template is read with the YAML reader of + parse_experiment_spec, which rejects duplicate keys, aliases, and merge + keys. Any other template is read with the sdl-yaml/v1 loader that + parse_sdl uses. It rejects duplicate keys, merge keys, and explicit tags, + and accepts acyclic aliases within its limits, as parse_sdl does. The root + mapping holds the template fields, so its keys are read literally; the + body below it gets the same structural key checks as an SDL document root. """ try: - raw = load_sdl_fragment(path.read_text(encoding="utf-8"), mapping_keys="literal") - except (SDLParseError, UnicodeDecodeError) as exc: + text = path.read_text(encoding="utf-8") + if contract == EXPERIMENT_CONTRACT: + raw = _read_experiment_yaml(text) + else: + raw = load_sdl_fragment(text, mapping_keys="literal") + except (SDLParseError, yaml.YAMLError, UnicodeDecodeError) as exc: return None, [_fail("example-library-parse", f"failed to parse {relative_path}: {exc}", relative_path)] if not isinstance(raw, dict): return None, [ From cdf228d71348451a583e92664e7072198a44b1ef Mon Sep 17 00:00:00 2001 From: Yernat Yestekov <2068106+doublewhy@users.noreply.github.com> Date: Mon, 12 Oct 2026 09:31:03 +0000 Subject: [PATCH 11/11] fix(examples): report an invalid date in an experiment template as a parse failure The experiment loader's YAML reader resolves YAML 1.1 timestamps and raises a bare ValueError on an unquoted date such as 2026-02-30. load_template_file caught only SDLParseError, yaml.YAMLError and UnicodeDecodeError, so check_example_library.py crashed with a traceback instead of reporting example-library-parse. Catch ValueError, which also covers UnicodeDecodeError. The new invalid-date case fails without this change. parse_experiment_spec lets the same ValueError through; #1563 tracks that. --- .../python/tests/test_issue_380_experiment_templates.py | 4 +++- tools/example_library_checks.py | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/implementations/python/tests/test_issue_380_experiment_templates.py b/implementations/python/tests/test_issue_380_experiment_templates.py index 8d474f5d9..09a312e6d 100644 --- a/implementations/python/tests/test_issue_380_experiment_templates.py +++ b/implementations/python/tests/test_issue_380_experiment_templates.py @@ -148,10 +148,12 @@ def test_experiment_contract_is_limited_to_run_and_study_templates( pytest.param(_TITLE_LINE, _TITLE_LINE + " title: Second title\n", id="duplicate-key"), pytest.param(_TEXT_LINES, " title: &text Policy test spec\n description: *text\n", id="scalar-alias"), pytest.param(" turn_order: sequential\n", " <<: {turn_order: sequential}\n", id="merge-key"), + # The reader resolves YAML 1.1 timestamps and raises a bare ValueError on this date. + pytest.param(_TEXT_LINES, _TITLE_LINE + " description: 2026-02-30\n", id="invalid-date"), ), ) def test_experiment_template_file_is_read_with_the_experiment_loader_reader(tmp_path: Path, old: str, new: str) -> None: - # yaml.safe_load accepts each edit, and the body it returns is a valid experiment spec. + # Except for invalid-date, yaml.safe_load accepts each edit and returns a valid experiment spec body. repo_root = _seed_experiment_repo(tmp_path) template_file = repo_root / _RUN_TEMPLATE_PATH source = template_file.read_text(encoding="utf-8") diff --git a/tools/example_library_checks.py b/tools/example_library_checks.py index af2fe5ab9..af859e9b3 100644 --- a/tools/example_library_checks.py +++ b/tools/example_library_checks.py @@ -170,7 +170,9 @@ def load_template_file( raw = _read_experiment_yaml(text) else: raw = load_sdl_fragment(text, mapping_keys="literal") - except (SDLParseError, yaml.YAMLError, UnicodeDecodeError) as exc: + # ValueError covers UnicodeDecodeError and the experiment reader's + # timestamp error on an invalid date such as 2026-02-30. + except (SDLParseError, yaml.YAMLError, ValueError) as exc: return None, [_fail("example-library-parse", f"failed to parse {relative_path}: {exc}", relative_path)] if not isinstance(raw, dict): return None, [