diff --git a/.github/workflows/validate-examples.yml b/.github/workflows/validate-examples.yml index c3dc265..45993d3 100644 --- a/.github/workflows/validate-examples.yml +++ b/.github/workflows/validate-examples.yml @@ -6,12 +6,14 @@ on: - "examples/**" - "references/**" - "scripts/validate_specula.py" + - "tests/**" - ".github/workflows/validate-examples.yml" pull_request: paths: - "examples/**" - "references/**" - "scripts/validate_specula.py" + - "tests/**" - ".github/workflows/validate-examples.yml" workflow_dispatch: @@ -30,9 +32,9 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - python -m pip install -r requirements.txt + python -m pip install -r requirements.txt pytest - - name: Validate every example (constitution + state machine + integration) + - name: Validate examples (strict for real, lenient for fictional) shell: bash run: | set -euo pipefail @@ -40,18 +42,20 @@ jobs: for dir in examples/*/; do const="${dir}constitution.json" sm="${dir}state-machine.json" - if [[ -f "$const" && -f "$sm" ]]; then - echo "::group::Validating ${dir}" - if ! python scripts/validate_specula.py \ - --constitution "$const" \ - --state-machine "$sm"; then - status=1 - fi - echo "::endgroup::" + [[ -f "$const" && -f "$sm" ]] || continue + # Real examples (metadata.fictional_example == false) must be warning-free. + fictional=$(python -c "import json,sys; print(json.load(open('$sm')).get('metadata',{}).get('fictional_example'))") + strict="" + if [[ "$fictional" == "False" ]]; then strict="--strict"; fi + echo "::group::Validating ${dir} (strict=${strict:-no})" + if ! python scripts/validate_specula.py $strict \ + --constitution "$const" --state-machine "$sm"; then + status=1 fi + echo "::endgroup::" done - if [[ "$status" -ne 0 ]]; then - echo "One or more examples failed schema/consistency validation." - exit 1 - fi - echo "All examples passed schema and consistency validation." + [[ "$status" -eq 0 ]] || { echo "Example validation failed."; exit 1; } + echo "All examples passed (real examples are warning-free)." + + - name: Run semantic coverage tests + run: pytest tests/ -q diff --git a/examples/community-space-brand/state-machine.json b/examples/community-space-brand/state-machine.json index f5b27c6..ab835ba 100644 --- a/examples/community-space-brand/state-machine.json +++ b/examples/community-space-brand/state-machine.json @@ -130,7 +130,11 @@ "id": "transition_state_to_generate", "from_state": "space_state_selection", "to_state": "generate_response", - "trigger": "space_state_selected" + "trigger": "space_state_selected", + "guard_ids": [ + "guard_relational_quality", + "guard_activation_color_ratio" + ] }, { "id": "transition_generate_to_ready", @@ -138,7 +142,8 @@ "to_state": "response_ready", "trigger": "response_validated", "guard_ids": [ - "guard_no_performance_pressure" + "guard_no_performance_pressure", + "guard_living_community" ] }, { @@ -163,7 +168,10 @@ "condition": "no_stigmatizing_language AND no_moralizing_tone AND people_before_labels", "principle": "principle_non_judgment", "action_on_violation": "block", - "severity": "high" + "severity": "high", + "constraint_ids": [ + "constraint_no_stigmatizing_language" + ] }, { "id": "guard_enabling", @@ -173,7 +181,10 @@ "condition": "response_provides_infrastructure OR response_provides_tools OR response_provides_access", "principle": "principle_enabling", "action_on_violation": "escalate", - "severity": "medium" + "severity": "medium", + "constraint_ids": [ + "constraint_no_ownership_asymmetry" + ] }, { "id": "guard_no_performance_pressure", @@ -183,7 +194,50 @@ "condition": "content_non_competitive AND no_achievement_framing AND no_ranking", "principle": "principle_imagination_in_practice", "action_on_violation": "retry", - "severity": "high" + "severity": "high", + "constraint_ids": [ + "constraint_no_performance_pressure" + ] + }, + { + "id": "guard_relational_quality", + "name": "Relational Quality Guard", + "applies_to": "transition", + "applies_at_state": "space_state_selection", + "condition": "attendance_optimization_does_not_reduce_relational_quality OR community_guardian_review_required", + "principle": "principle_habitability", + "constraint_ids": [ + "constraint_relational_quality_over_attendance" + ], + "action_on_violation": "escalate", + "severity": "medium", + "error_message": "Attendance cannot be optimized at the expense of quality of shared presence." + }, + { + "id": "guard_activation_color_ratio", + "name": "Activation Color Ratio Guard", + "applies_to": "output", + "applies_at_state": "space_state_selection", + "condition": "red_color_ratio <= 0.05", + "principle": "principle_habitability", + "constraint_ids": [ + "constraint_activation_color_ratio" + ], + "action_on_violation": "retry", + "severity": "high", + "error_message": "Red activation color must not exceed five percent of visual or spatial presence." + }, + { + "id": "guard_living_community", + "name": "Living Community Continuity Guard", + "applies_to": "transition", + "applies_at_state": "generate_response", + "condition": "response_supports_continuity_or_relationship_depth", + "principle": "principle_living_community", + "constraint_ids": [], + "action_on_violation": "escalate", + "severity": "medium", + "error_message": "The response must support continuity, trust, or relationship depth rather than one-off consumption." } ], "space_states": { diff --git a/examples/luxury-fashion-brand/state-machine.json b/examples/luxury-fashion-brand/state-machine.json index 5f8e3e7..d49ab06 100644 --- a/examples/luxury-fashion-brand/state-machine.json +++ b/examples/luxury-fashion-brand/state-machine.json @@ -9,43 +9,61 @@ "id": "receive_customer_inquiry", "name": "Receive Customer Inquiry", "type": "normal", - "allowed_transitions": ["classify_customer_tier"] + "allowed_transitions": [ + "classify_customer_tier" + ] }, "classify_customer_tier": { "id": "classify_customer_tier", "name": "Classify Customer Tier", "type": "normal", - "allowed_transitions": ["brand_voice_alignment_check"] + "allowed_transitions": [ + "brand_voice_alignment_check" + ] }, "brand_voice_alignment_check": { "id": "brand_voice_alignment_check", "name": "Brand Voice Alignment Check", "type": "normal", - "allowed_transitions": ["heritage_compatibility_check", "escalate_brand_guardian"] + "allowed_transitions": [ + "heritage_compatibility_check", + "escalate_brand_guardian" + ] }, "heritage_compatibility_check": { "id": "heritage_compatibility_check", "name": "Heritage Compatibility Check", "type": "normal", - "allowed_transitions": ["personalization_depth_selection", "escalate_brand_guardian"] + "allowed_transitions": [ + "personalization_depth_selection", + "escalate_brand_guardian" + ] }, "personalization_depth_selection": { "id": "personalization_depth_selection", "name": "Personalization Depth Selection", "type": "normal", - "allowed_transitions": ["generate_response", "escalate_brand_guardian"] + "allowed_transitions": [ + "generate_response", + "escalate_brand_guardian" + ] }, "generate_response": { "id": "generate_response", "name": "Generate Response", "type": "normal", - "allowed_transitions": ["response_ready", "escalate_brand_guardian"] + "allowed_transitions": [ + "response_ready", + "escalate_brand_guardian" + ] }, "escalate_brand_guardian": { "id": "escalate_brand_guardian", "name": "Escalate to Brand Guardian", "type": "escalation", - "allowed_transitions": ["response_ready"] + "allowed_transitions": [ + "response_ready" + ] }, "response_ready": { "id": "response_ready", @@ -71,21 +89,27 @@ "from_state": "brand_voice_alignment_check", "to_state": "heritage_compatibility_check", "trigger": "voice_guard_passed", - "guard_ids": ["guard_brand_voice_alignment"] + "guard_ids": [ + "guard_brand_voice_alignment" + ] }, { "id": "transition_heritage_to_personalization", "from_state": "heritage_compatibility_check", "to_state": "personalization_depth_selection", "trigger": "heritage_guard_passed", - "guard_ids": ["guard_heritage_alignment"] + "guard_ids": [ + "guard_heritage_alignment" + ] }, { "id": "transition_personalization_to_generate", "from_state": "personalization_depth_selection", "to_state": "generate_response", "trigger": "exclusivity_guard_passed", - "guard_ids": ["guard_exclusivity_enforcement"] + "guard_ids": [ + "guard_exclusivity_enforcement" + ] }, { "id": "transition_generate_to_ready", @@ -110,7 +134,10 @@ "condition": "brand_voice_compliant == true", "principle": "principle_brand_voice", "action_on_violation": "block", - "severity": "high" + "severity": "high", + "constraint_ids": [ + "constraint_brand_tone" + ] }, { "id": "guard_heritage_alignment", @@ -120,7 +147,10 @@ "condition": "heritage_alignment_passed == true", "principle": "principle_heritage_respect", "action_on_violation": "escalate", - "severity": "critical" + "severity": "critical", + "constraint_ids": [ + "constraint_heritage_consistency" + ] }, { "id": "guard_exclusivity_enforcement", @@ -130,7 +160,11 @@ "condition": "personalization_depth >= required_depth", "principle": "principle_customer_exclusivity", "action_on_violation": "escalate", - "severity": "high" + "severity": "high", + "constraint_ids": [ + "constraint_core_collection_premium", + "constraint_vip_personalization_depth" + ] } ], "metadata": { diff --git a/references/schemas-statemachine.json b/references/schemas-statemachine.json index 5159e85..c958d2e 100644 --- a/references/schemas-statemachine.json +++ b/references/schemas-statemachine.json @@ -3,7 +3,11 @@ "title": "SPECULA State Machine Schema", "description": "Formal specification for state machines implementing SPECULA governance", "type": "object", - "required": ["id", "initial_state", "states"], + "required": [ + "id", + "initial_state", + "states" + ], "properties": { "id": { "type": "string", @@ -30,7 +34,10 @@ "description": "Map of state_id -> state definition", "additionalProperties": { "type": "object", - "required": ["id", "name"], + "required": [ + "id", + "name" + ], "properties": { "id": { "type": "string", @@ -41,7 +48,12 @@ }, "type": { "type": "string", - "enum": ["normal", "terminal", "error", "escalation"], + "enum": [ + "normal", + "terminal", + "error", + "escalation" + ], "description": "normal=intermediate, terminal=success end state, error=failure state, escalation=human review" }, "description": { @@ -49,7 +61,9 @@ }, "entry_actions": { "type": "array", - "items": {"type": "string"}, + "items": { + "type": "string" + }, "description": "Actions executed when entering state" }, "exit_guards": { @@ -57,12 +71,22 @@ "items": { "type": "object", "properties": { - "id": {"type": "string"}, - "condition": {"type": "string"}, - "on_fail": {"type": "string"}, + "id": { + "type": "string" + }, + "condition": { + "type": "string" + }, + "on_fail": { + "type": "string" + }, "severity": { "type": "string", - "enum": ["block", "warn", "log"] + "enum": [ + "block", + "warn", + "log" + ] } } }, @@ -70,14 +94,20 @@ }, "allowed_transitions": { "type": "array", - "items": {"type": "string"}, + "items": { + "type": "string" + }, "description": "Which states can be transitioned to from here" }, "timeout": { "type": "object", "properties": { - "duration_ms": {"type": "number"}, - "on_timeout": {"type": "string"} + "duration_ms": { + "type": "number" + }, + "on_timeout": { + "type": "string" + } }, "description": "Optional timeout and fallback action" } @@ -89,7 +119,11 @@ "description": "Explicit transition definitions", "items": { "type": "object", - "required": ["id", "from_state", "to_state"], + "required": [ + "id", + "from_state", + "to_state" + ], "properties": { "id": { "type": "string", @@ -107,17 +141,23 @@ }, "preconditions": { "type": "array", - "items": {"type": "string"}, + "items": { + "type": "string" + }, "description": "Conditions that must be true" }, "actions": { "type": "array", - "items": {"type": "string"}, + "items": { + "type": "string" + }, "description": "Actions to execute during transition" }, "guard_ids": { "type": "array", - "items": {"type": "string"}, + "items": { + "type": "string" + }, "description": "Guard checks that must pass" }, "is_fallback": { @@ -132,16 +172,25 @@ "description": "Constitutional guard conditions", "items": { "type": "object", - "required": ["id", "condition"], + "required": [ + "id", + "condition" + ], "properties": { "id": { "type": "string", "pattern": "^guard_[a-z0-9_]+$" }, - "name": {"type": "string"}, + "name": { + "type": "string" + }, "applies_to": { "type": "string", - "enum": ["state", "transition", "output"], + "enum": [ + "state", + "transition", + "output" + ], "description": "Where this guard applies" }, "applies_at_state": { @@ -158,15 +207,34 @@ }, "action_on_violation": { "type": "string", - "enum": ["block", "escalate", "log", "retry"], + "enum": [ + "block", + "escalate", + "log", + "retry" + ], "description": "What happens if violated" }, "severity": { "type": "string", - "enum": ["critical", "high", "medium", "low"] + "enum": [ + "critical", + "high", + "medium", + "low" + ] }, "error_message": { "type": "string" + }, + "constraint_ids": { + "type": "array", + "description": "Constitutional constraints this guard directly enforces", + "items": { + "type": "string", + "pattern": "^constraint_[a-z0-9_]+$" + }, + "uniqueItems": true } } } @@ -176,19 +244,26 @@ "description": "Reusable action definitions", "items": { "type": "object", - "required": ["id", "implementation"], + "required": [ + "id", + "implementation" + ], "properties": { "id": { "type": "string" }, - "name": {"type": "string"}, + "name": { + "type": "string" + }, "implementation": { "type": "string", "description": "How to execute this action" }, "parameters": { "type": "array", - "items": {"type": "string"} + "items": { + "type": "string" + } } } } @@ -214,21 +289,35 @@ "items": { "type": "object", "properties": { - "constraint": {"type": "string"}, - "applies_globally": {"type": "boolean"} + "constraint": { + "type": "string" + }, + "applies_globally": { + "type": "boolean" + } } } }, "metadata": { "type": "object", "properties": { - "author": {"type": "string"}, - "created_timestamp": {"type": "string"}, - "last_modified": {"type": "string"}, - "framework_version": {"type": "string"}, + "author": { + "type": "string" + }, + "created_timestamp": { + "type": "string" + }, + "last_modified": { + "type": "string" + }, + "framework_version": { + "type": "string" + }, "references": { "type": "array", - "items": {"type": "string"} + "items": { + "type": "string" + } } } } diff --git a/scripts/validate_specula.py b/scripts/validate_specula.py index 0ffffde..427f5dc 100644 --- a/scripts/validate_specula.py +++ b/scripts/validate_specula.py @@ -40,8 +40,9 @@ class SPECULAValidator: """Validates SPECULA governance structures.""" - def __init__(self, verbose: bool = False): + def __init__(self, verbose: bool = False, strict: bool = False): self.verbose = verbose + self.strict = strict self.errors: List[str] = [] self.warnings: List[str] = [] self._schema_warning_emitted = False @@ -289,6 +290,20 @@ def validate_integration(self, constitution_path: str, sm_path: str) -> bool: f"Constraint {constraint.get('id')} principle '{principle}' has no mapped guard" ) + # Direct constraint -> guard coverage via guard.constraint_ids. + # Stronger than principle-level mapping: proves each declared constraint + # has an identifiable enforcement rule, not just a shared principle. + constraint_ids = {c.get("id") for c in constitution.get("constraints", []) if c.get("id")} + mapped_constraints = { + constraint_id + for guard in sm.get("guards", []) + for constraint_id in guard.get("constraint_ids", []) + } + for constraint_id in sorted(constraint_ids - mapped_constraints): + self.warnings.append( + f"Constraint {constraint_id} has no directly mapped guard" + ) + return len(self.errors) == 0 def validate_workflow(self, workflow_path: str) -> bool: @@ -418,7 +433,12 @@ def print_report(self): if not self.errors and not self.warnings: print("\n✅ All validations passed!") + if self.strict and self.warnings and not self.errors: + print("\n❌ STRICT MODE: warnings are treated as failures for this example.") + print("\n" + "=" * 70) + if self.strict: + return not self.errors and not self.warnings return len(self.errors) == 0 @@ -432,10 +452,15 @@ def main(): "Validates phase sequence, prerequisites, and dual-role requirements.", ) parser.add_argument("--verbose", "-v", action="store_true", help="Verbose output") + parser.add_argument( + "--strict", + action="store_true", + help="Treat warnings as failures (recommended for real, non-fictional examples).", + ) args = parser.parse_args() - validator = SPECULAValidator(verbose=args.verbose) + validator = SPECULAValidator(verbose=args.verbose, strict=args.strict) # Run validations valid = True @@ -456,9 +481,9 @@ def main(): if not validator.validate_workflow(args.workflow): valid = False - validator.print_report() + report_ok = validator.print_report() - sys.exit(0 if valid else 1) + sys.exit(0 if (valid and report_ok) else 1) if __name__ == "__main__": diff --git a/tests/test_semantic_coverage.py b/tests/test_semantic_coverage.py new file mode 100644 index 0000000..0f469d8 --- /dev/null +++ b/tests/test_semantic_coverage.py @@ -0,0 +1,88 @@ +"""Semantic coverage tests for SPECULA skill examples. + +These go beyond JSON-schema validity: they assert that every constitutional +principle and constraint in the *real* examples is actually enforced by a guard +that is attached to the runtime flow (a transition). This is the +principle -> constraint -> guard -> transition chain the skill promises. +""" +import importlib.util +import json +from pathlib import Path + +import pytest + +REPO = Path(__file__).resolve().parent.parent +EXAMPLES = REPO / "examples" + +# Load the validator module directly from scripts/ (no package install needed). +_spec = importlib.util.spec_from_file_location( + "validate_specula", REPO / "scripts" / "validate_specula.py" +) +validate_specula = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(validate_specula) +SPECULAValidator = validate_specula.SPECULAValidator + + +def _example_dirs(): + return sorted(p for p in EXAMPLES.iterdir() + if p.is_dir() and (p / "constitution.json").exists() + and (p / "state-machine.json").exists()) + + +def _run_validation(example_dir: Path): + v = SPECULAValidator() + const = str(example_dir / "constitution.json") + sm = str(example_dir / "state-machine.json") + v.validate_constitution(const) + v.validate_state_machine(sm) + v.validate_integration(const, sm) + return v + + +def _is_real(example_dir: Path) -> bool: + sm = json.loads((example_dir / "state-machine.json").read_text(encoding="utf-8")) + return sm.get("metadata", {}).get("fictional_example") is False + + +REAL_EXAMPLES = [p for p in _example_dirs() if _is_real(p)] + + +@pytest.mark.parametrize("example_dir", REAL_EXAMPLES, ids=lambda p: p.name) +def test_all_constitutional_principles_have_guards(example_dir): + v = _run_validation(example_dir) + offenders = [w for w in v.warnings if "not implemented as guard" in w] + assert not offenders, f"{example_dir.name}: {offenders}" + + +@pytest.mark.parametrize("example_dir", REAL_EXAMPLES, ids=lambda p: p.name) +def test_all_constraints_have_direct_guard_mapping(example_dir): + v = _run_validation(example_dir) + offenders = [w for w in v.warnings + if "no directly mapped guard" in w or "has no mapped guard" in w] + assert not offenders, f"{example_dir.name}: {offenders}" + + +@pytest.mark.parametrize("example_dir", REAL_EXAMPLES, ids=lambda p: p.name) +def test_real_examples_have_zero_warnings(example_dir): + v = _run_validation(example_dir) + assert not v.errors, f"{example_dir.name} errors: {v.errors}" + assert not v.warnings, f"{example_dir.name} warnings: {v.warnings}" + + +def test_habitability_guards_are_attached_to_runtime_flow(): + sm = json.loads( + (EXAMPLES / "community-space-brand" / "state-machine.json").read_text(encoding="utf-8") + ) + attached = { + guard_id + for transition in sm["transitions"] + for guard_id in transition.get("guard_ids", []) + } + assert "guard_relational_quality" in attached + assert "guard_activation_color_ratio" in attached + assert "guard_living_community" in attached + + +def test_real_examples_exist(): + # Guard against silently testing nothing if the metadata flag changes. + assert REAL_EXAMPLES, "no real (non-fictional) examples found to test"