From ec6fbea166bb205e97118a0549e716011a407dd5 Mon Sep 17 00:00:00 2001 From: Michael Heller <21163552+mdheller@users.noreply.github.com> Date: Fri, 31 Jul 2026 20:59:01 -0400 Subject: [PATCH] feat(schemas): Twin World-Model & GAIA + EvalItem schema pack Give a Twin a dynamical, geospatially-grounded world model beneath its governance lifecycle, and add the EvalItem contract for the provider-neutral model tournament that feeds the Intelligence-Superiority Bench (iSOTA). World-model + GAIA pack (6 schemas): WorldModel (x+ = A*x + B*(G*u)), StateVector, ImpulseGate, Impulse, Region, GaiaObservation. GAIA is the exogenous world model: a Region is grounded in GAIA cells, an exogenous_shock Impulse must cite a GaiaObservation, and a WorldModel that admits exogenous shocks must declare exogenous_source=gaia -- fail-closed, asserted by rejection. 16 rejection invariants (incl. exogenous-shock=>GAIA, closed-gate=>no-admit, state components in [0,1]). EvalItem: the atomic contract of the Provider Eval Seed Strategy -- corpus A/B/C, task family, expected traits, grading method, risk class. Ungradeable items and provider-sourced items with no provider named are rejected; the harness stays provider-neutral. 6 rejection invariants. Both validators wired into `make validate`; examples carry the `type` discriminator for the Validate-Spec example mapper. --- Makefile | 12 ++- examples/eval-item.json | 18 ++++ examples/gaia-observation.json | 11 +++ examples/impulse-gate.json | 9 ++ examples/impulse.json | 13 +++ examples/region.json | 8 ++ examples/state-vector.json | 8 ++ examples/world-model.json | 19 +++++ schemas/EvalItem.json | 40 +++++++++ schemas/GaiaObservation.json | 31 +++++++ schemas/Impulse.json | 38 +++++++++ schemas/ImpulseGate.json | 31 +++++++ schemas/Region.json | 36 ++++++++ schemas/StateVector.json | 21 +++++ schemas/WorldModel.json | 52 ++++++++++++ tools/validate_eval_item_examples.py | 67 +++++++++++++++ tools/validate_world_model_examples.py | 112 +++++++++++++++++++++++++ 17 files changed, 524 insertions(+), 2 deletions(-) create mode 100644 examples/eval-item.json create mode 100644 examples/gaia-observation.json create mode 100644 examples/impulse-gate.json create mode 100644 examples/impulse.json create mode 100644 examples/region.json create mode 100644 examples/state-vector.json create mode 100644 examples/world-model.json create mode 100644 schemas/EvalItem.json create mode 100644 schemas/GaiaObservation.json create mode 100644 schemas/Impulse.json create mode 100644 schemas/ImpulseGate.json create mode 100644 schemas/Region.json create mode 100644 schemas/StateVector.json create mode 100644 schemas/WorldModel.json create mode 100644 tools/validate_eval_item_examples.py create mode 100644 tools/validate_world_model_examples.py diff --git a/Makefile b/Makefile index 9b74c50..ff05456 100644 --- a/Makefile +++ b/Makefile @@ -1,12 +1,20 @@ -.PHONY: validate validate-resource-contract validate-measurement validate-value-type validate-source-locator validate-sourceos-repo-manifest validate-mesh-action-registry validate-control-plane-examples validate-nlboot-examples validate-lattice-data-governai-examples validate-ops-history-examples validate-runtime-observability-examples validate-interpretability-examples validate-lifecycle-boundary-examples validate-svf-contracts validate-sync-cycle-receipts validate-onboarding-examples validate-runtime-causality-examples validate-agentic-os-examples validate-triparty-examples validate-labor-market-examples validate-supply-chain-risk-examples validate-reasoning-examples validate-mpcc-event-examples validate-knowledge-nugget-examples validate-semantic-action-examples validate-epistemic-kernel-examples validate-ab-update-examples validate-device-service-examples validate-duplicate-schema-ids validate-lawful-dispatch-receipt validate-architectural-building-block validate-agent-passport-examples validate-seam-definition-examples validate-agent-system-vocabulary validate-genesis-inception-examples validate-measurement +.PHONY: validate validate-resource-contract validate-measurement validate-value-type validate-source-locator validate-sourceos-repo-manifest validate-mesh-action-registry validate-control-plane-examples validate-nlboot-examples validate-lattice-data-governai-examples validate-ops-history-examples validate-runtime-observability-examples validate-interpretability-examples validate-lifecycle-boundary-examples validate-svf-contracts validate-sync-cycle-receipts validate-onboarding-examples validate-runtime-causality-examples validate-agentic-os-examples validate-triparty-examples validate-labor-market-examples validate-supply-chain-risk-examples validate-reasoning-examples validate-mpcc-event-examples validate-knowledge-nugget-examples validate-semantic-action-examples validate-epistemic-kernel-examples validate-ab-update-examples validate-device-service-examples validate-duplicate-schema-ids validate-lawful-dispatch-receipt validate-architectural-building-block validate-agent-passport-examples validate-seam-definition-examples validate-agent-system-vocabulary validate-genesis-inception-examples validate-measurement validate-world-model-examples validate-eval-item-examples -validate: validate-control-plane-examples validate-nlboot-examples validate-lattice-data-governai-examples validate-ops-history-examples validate-runtime-observability-examples validate-interpretability-examples validate-lifecycle-boundary-examples validate-svf-contracts validate-sync-cycle-receipts validate-onboarding-examples validate-runtime-causality-examples validate-agentic-os-examples validate-triparty-examples validate-labor-market-examples validate-supply-chain-risk-examples validate-reasoning-examples validate-mpcc-event-examples validate-knowledge-nugget-examples validate-semantic-action-examples validate-epistemic-kernel-examples validate-ab-update-examples validate-device-service-examples validate-duplicate-schema-ids validate-value-type validate-source-locator validate-sourceos-repo-manifest validate-mesh-action-registry validate-lawful-dispatch-receipt validate-architectural-building-block validate-agent-passport-examples validate-seam-definition-examples validate-agent-system-vocabulary validate-genesis-inception-examples validate-measurement validate-resource-contract +validate: validate-control-plane-examples validate-nlboot-examples validate-lattice-data-governai-examples validate-ops-history-examples validate-runtime-observability-examples validate-interpretability-examples validate-lifecycle-boundary-examples validate-svf-contracts validate-sync-cycle-receipts validate-onboarding-examples validate-runtime-causality-examples validate-agentic-os-examples validate-triparty-examples validate-labor-market-examples validate-supply-chain-risk-examples validate-reasoning-examples validate-mpcc-event-examples validate-knowledge-nugget-examples validate-semantic-action-examples validate-epistemic-kernel-examples validate-ab-update-examples validate-device-service-examples validate-duplicate-schema-ids validate-value-type validate-source-locator validate-sourceos-repo-manifest validate-mesh-action-registry validate-lawful-dispatch-receipt validate-architectural-building-block validate-agent-passport-examples validate-seam-definition-examples validate-agent-system-vocabulary validate-genesis-inception-examples validate-measurement validate-world-model-examples validate-eval-item-examples validate-resource-contract @echo "OK: validate" validate-genesis-inception-examples: python3 -m pip install --user jsonschema >/dev/null python3 tools/validate_genesis_inception_examples.py +validate-world-model-examples: + python3 -m pip install --user jsonschema >/dev/null + python3 tools/validate_world_model_examples.py + +validate-eval-item-examples: + python3 -m pip install --user jsonschema >/dev/null + python3 tools/validate_eval_item_examples.py + validate-agent-system-vocabulary: python3 tools/validate_agent_system_vocabulary.py diff --git a/examples/eval-item.json b/examples/eval-item.json new file mode 100644 index 0000000..9ff6cac --- /dev/null +++ b/examples/eval-item.json @@ -0,0 +1,18 @@ +{ + "type": "EvalItem", + "id": "ev:B/evidence_answer/00123", + "source": "internal", + "corpus": "B", + "task_family": "evidence_answer", + "user_question": "A customer reports that release 2026.7 broke SSO for their Okta tenant. Given the linked incident and changelog, what is the root cause and the exact remediation step?", + "context_doc_ids": ["inc:2026-07-29/sso-okta", "changelog:2026.7", "kb:okta-saml-clock-skew"], + "expected_answer_traits": [ + "names the SAML clock-skew tolerance change in 2026.7 as the cause", + "cites the changelog entry and the incident", + "gives the exact config remediation, not a generic 'contact support'", + "does not fabricate a CVE or a version that does not appear in context" + ], + "gold_citations": ["changelog:2026.7#saml-clock-skew", "inc:2026-07-29/sso-okta#rootcause"], + "grading_method": "citation_faithfulness", + "risk_class": "high" +} diff --git a/examples/gaia-observation.json b/examples/gaia-observation.json new file mode 100644 index 0000000..25ff5f4 --- /dev/null +++ b/examples/gaia-observation.json @@ -0,0 +1,11 @@ +{ + "type": "GaiaObservation", + "observation_id": "gaia:obs/2026-07-31T12:00Z/8928308280fffff/hazard_index", + "gaia_cell": { "system": "h3", "cell": "8928308280fffff" }, + "field": "hazard_index", + "value": 0.62, + "unit": "index_0_1", + "valid_time": "2026-07-31T12:00:00Z", + "source": "twc", + "provenance_refs": ["gaia:ingest/twc/2026-07-31T12:00Z", "receipt:compute-gateway/materialize/9f2a"] +} diff --git a/examples/impulse-gate.json b/examples/impulse-gate.json new file mode 100644 index 0000000..22485d0 --- /dev/null +++ b/examples/impulse-gate.json @@ -0,0 +1,9 @@ +{ + "type": "ImpulseGate", + "gate_id": "gate:twn_mkt-0001/exogenous_shock", + "impulse_class": "exogenous_shock", + "mode": "attenuated", + "gate_factor": 0.35, + "fail_closed": true, + "admission_rule": "GAIA hazard_index shocks admitted at 0.35 unless the twin is verified and the region is under an active advisory, in which case escalate to human_action gate." +} diff --git a/examples/impulse.json b/examples/impulse.json new file mode 100644 index 0000000..fc8792f --- /dev/null +++ b/examples/impulse.json @@ -0,0 +1,13 @@ +{ + "type": "Impulse", + "impulse_id": "imp:twn_mkt-0001/000042", + "twin_id": "twn_mkt-0001", + "impulse_class": "exogenous_shock", + "source": "gaia", + "magnitude": 0.62, + "target_dims": ["risk", "drift"], + "gate_state": "attenuated", + "admitted": true, + "source_ref": "gaia:obs/2026-07-31T12:00Z/8928308280fffff/hazard_index", + "provenance_refs": ["gate:twn_mkt-0001/exogenous_shock", "gaia:obs/2026-07-31T12:00Z/8928308280fffff/hazard_index"] +} diff --git a/examples/region.json b/examples/region.json new file mode 100644 index 0000000..154dafe --- /dev/null +++ b/examples/region.json @@ -0,0 +1,8 @@ +{ + "type": "Region", + "region_id": "region:au/sydney-ashore", + "name": "Sydney · ASX", + "centroid": { "lat": -33.87, "lon": 151.21 }, + "bbox": [150.5, -34.2, 151.6, -33.5], + "gaia_cells": ["8928308280fffff", "8928308281fffff", "89283082837ffff"] +} diff --git a/examples/state-vector.json b/examples/state-vector.json new file mode 100644 index 0000000..b725fc2 --- /dev/null +++ b/examples/state-vector.json @@ -0,0 +1,8 @@ +{ + "type": "StateVector", + "twin_id": "twn_mkt-0001", + "seq": 42, + "at": "2026-07-31T12:00:03Z", + "dims": { "integrity": 0.88, "risk": 0.61, "load": 0.44, "coverage": 0.72, "drift": 0.39 }, + "provenance_refs": ["imp:twn_mkt-0001/000042"] +} diff --git a/examples/world-model.json b/examples/world-model.json new file mode 100644 index 0000000..c793cba --- /dev/null +++ b/examples/world-model.json @@ -0,0 +1,19 @@ +{ + "type": "WorldModel", + "model_id": "wm:twn_mkt-0001", + "twin_id": "twn_mkt-0001", + "state_dims": ["integrity", "risk", "load", "coverage", "drift"], + "dynamics": { + "relaxation": 0.90, + "baseline": { "integrity": 0.85, "risk": 0.30, "load": 0.40, "coverage": 0.70, "drift": 0.20 } + }, + "gates": [ + { "impulse_class": "device_reading", "mode": "open" }, + { "impulse_class": "market_data_event", "mode": "open" }, + { "impulse_class": "exogenous_shock", "mode": "attenuated" }, + { "impulse_class": "policy_change", "mode": "closed" }, + { "impulse_class": "human_action", "mode": "open" } + ], + "region_ref": "region:au/sydney-ashore", + "exogenous_source": "gaia" +} diff --git a/schemas/EvalItem.json b/schemas/EvalItem.json new file mode 100644 index 0000000..2d0a243 --- /dev/null +++ b/schemas/EvalItem.json @@ -0,0 +1,40 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://schemas.srcos.ai/v2/EvalItem.json", + "title": "EvalItem", + "description": "One item in the provider-neutral model-tournament corpora that feed the Intelligence-Superiority Bench (iSOTA). An EvalItem is the atomic contract of the Provider Eval Seed Strategy: it declares which corpus it belongs to (A provider-seed / B Sherlock-task / C adversarial), its task family, the question, the traits an answer must have, how it is graded, and its risk class. The load-bearing rule is gradability — an item with no expected_answer_traits cannot be scored and is rejected. Provider-sourced items must name the provider; the harness stays neutral, so provider is a label, never a winner.", + "type": "object", + "additionalProperties": false, + "required": ["type", "id", "source", "corpus", "task_family", "user_question", "expected_answer_traits", "grading_method", "risk_class"], + "properties": { + "type": { "const": "EvalItem" }, + "id": { "type": "string", "minLength": 1 }, + "source": { "type": "string", "enum": ["provider", "internal"] }, + "provider": { "type": "string" }, + "corpus": { + "type": "string", + "enum": ["A", "B", "C"], + "description": "A = provider seed (screening); B = Sherlock task (our workload, weighted heavy); C = adversarial / edge." + }, + "task_family": { + "type": "string", + "enum": ["support_triage", "duplicate_detection", "evidence_answer", "incident_timeline", "routing_escalation", "repo_diagnosis", "release_impact", "prompt_eng", "rag_grounding", "tool_use", "structured_output", "safety_policy", "adversarial"] + }, + "user_question": { "type": "string", "minLength": 1 }, + "context_doc_ids": { "type": "array", "items": { "type": "string" } }, + "expected_answer_traits": { "type": "array", "items": { "type": "string", "minLength": 1 }, "minItems": 1 }, + "gold_answer": { "type": "string" }, + "gold_citations": { "type": "array", "items": { "type": "string" } }, + "grading_method": { + "type": "string", + "enum": ["exact", "citation_faithfulness", "groundedness", "instruction_following", "tool_accuracy", "case_action", "human", "llm_judge"] + }, + "risk_class": { "type": "string", "enum": ["low", "medium", "high", "critical"] } + }, + "allOf": [ + { + "if": { "properties": { "source": { "const": "provider" } } }, + "then": { "required": ["provider"] } + } + ] +} diff --git a/schemas/GaiaObservation.json b/schemas/GaiaObservation.json new file mode 100644 index 0000000..8558fc4 --- /dev/null +++ b/schemas/GaiaObservation.json @@ -0,0 +1,31 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://schemas.srcos.ai/v2/GaiaObservation.json", + "title": "GaiaObservation", + "description": "An exogenous environmental world-state reading from GAIA — the estate's weather/environmental world model (TWC / IBM alt-data). A GaiaObservation is the grounding for the exogenous state that drives a twin's WorldModel: an ExogenousShock Impulse must cite one, and a Region is grounded in the GAIA cells its observations cover. Provenance is load-bearing — an ungrounded observation is inadmissible (asserted by rejection).", + "type": "object", + "additionalProperties": false, + "required": ["type", "observation_id", "gaia_cell", "field", "value", "valid_time", "source", "provenance_refs"], + "properties": { + "type": { "const": "GaiaObservation" }, + "observation_id": { "type": "string", "minLength": 1 }, + "gaia_cell": { + "type": "object", + "additionalProperties": false, + "required": ["system", "cell"], + "properties": { + "system": { "type": "string", "enum": ["h3", "latlon_grid", "s2"] }, + "cell": { "type": "string", "minLength": 1 } + } + }, + "field": { + "type": "string", + "enum": ["temperature", "precipitation", "wind_speed", "humidity", "pressure", "air_quality", "hazard_index", "flood_risk", "wildfire_risk", "solar_irradiance"] + }, + "value": { "type": "number" }, + "unit": { "type": "string" }, + "valid_time": { "type": "string", "format": "date-time" }, + "source": { "type": "string", "enum": ["twc", "ibm_environmental", "noaa", "ecmwf", "synthetic"] }, + "provenance_refs": { "type": "array", "items": { "type": "string" }, "minItems": 1 } + } +} diff --git a/schemas/Impulse.json b/schemas/Impulse.json new file mode 100644 index 0000000..6430eab --- /dev/null +++ b/schemas/Impulse.json @@ -0,0 +1,38 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://schemas.srcos.ai/v2/Impulse.json", + "title": "Impulse", + "description": "A typed input to a twin's state-space world model: u in x+ = A*x + B*(G*u). An Impulse records the class, source, magnitude, which state dimensions it targets, and — after the gate has run — whether it was admitted and the gate state that decided it. Two invariants are load-bearing and asserted by rejection: an exogenous_shock MUST cite the GaiaObservation it came from (GAIA is the exogenous world model), and an impulse admitted through a closed gate is a contradiction (fail-closed).", + "type": "object", + "additionalProperties": false, + "required": ["type", "impulse_id", "twin_id", "impulse_class", "source", "magnitude", "target_dims", "gate_state", "admitted", "provenance_refs"], + "properties": { + "type": { "const": "Impulse" }, + "impulse_id": { "type": "string", "minLength": 1 }, + "twin_id": { "type": "string", "minLength": 1 }, + "impulse_class": { + "type": "string", + "enum": ["device_reading", "market_data_event", "exogenous_shock", "policy_change", "human_action"] + }, + "source": { "type": "string", "minLength": 1 }, + "magnitude": { "type": "number", "minimum": 0, "maximum": 1 }, + "target_dims": { "type": "array", "items": { "type": "string" }, "minItems": 1 }, + "gate_state": { "type": "string", "enum": ["open", "attenuated", "closed"] }, + "admitted": { "type": "boolean" }, + "source_ref": { + "type": "string", + "description": "For exogenous_shock: the GaiaObservation.observation_id this impulse was derived from." + }, + "provenance_refs": { "type": "array", "items": { "type": "string" }, "minItems": 1 } + }, + "allOf": [ + { + "if": { "properties": { "impulse_class": { "const": "exogenous_shock" } } }, + "then": { "required": ["source_ref"] } + }, + { + "if": { "properties": { "gate_state": { "const": "closed" } } }, + "then": { "properties": { "admitted": { "const": false } } } + } + ] +} diff --git a/schemas/ImpulseGate.json b/schemas/ImpulseGate.json new file mode 100644 index 0000000..498afaa --- /dev/null +++ b/schemas/ImpulseGate.json @@ -0,0 +1,31 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://schemas.srcos.ai/v2/ImpulseGate.json", + "title": "ImpulseGate", + "description": "A governed admission gate for one impulse class into a twin's state-space dynamics. The gate is where governance meets dynamics: it decides whether an input of its class may perturb the state vector, and by how much (gate_factor). Fail-closed is the default and the load-bearing rule — a closed gate must have gate_factor 0, so an ungated impulse cannot move the state (asserted by rejection).", + "type": "object", + "additionalProperties": false, + "required": ["type", "gate_id", "impulse_class", "mode", "gate_factor"], + "properties": { + "type": { "const": "ImpulseGate" }, + "gate_id": { "type": "string", "minLength": 1 }, + "impulse_class": { + "type": "string", + "enum": ["device_reading", "market_data_event", "exogenous_shock", "policy_change", "human_action"] + }, + "mode": { "type": "string", "enum": ["open", "attenuated", "closed"] }, + "gate_factor": { "type": "number", "minimum": 0, "maximum": 1 }, + "fail_closed": { "type": "boolean", "default": true }, + "admission_rule": { "type": "string" } + }, + "allOf": [ + { + "if": { "properties": { "mode": { "const": "closed" } } }, + "then": { "properties": { "gate_factor": { "const": 0 } } } + }, + { + "if": { "properties": { "mode": { "const": "open" } } }, + "then": { "properties": { "gate_factor": { "const": 1 } } } + } + ] +} diff --git a/schemas/Region.json b/schemas/Region.json new file mode 100644 index 0000000..8e628d7 --- /dev/null +++ b/schemas/Region.json @@ -0,0 +1,36 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://schemas.srcos.ai/v2/Region.json", + "title": "Region", + "description": "A geospatial region a twin is grounded in. A Region is the join between the twin's WorldModel and GAIA: it carries a centroid for the Common Operating Picture and is grounded in one or more GAIA cells, which is where its exogenous (weather/environmental) state is read from. An ungrounded region — no GAIA cells — cannot supply exogenous state and is rejected.", + "type": "object", + "additionalProperties": false, + "required": ["type", "region_id", "name", "centroid", "gaia_cells"], + "properties": { + "type": { "const": "Region" }, + "region_id": { "type": "string", "minLength": 1 }, + "name": { "type": "string", "minLength": 1 }, + "centroid": { + "type": "object", + "additionalProperties": false, + "required": ["lat", "lon"], + "properties": { + "lat": { "type": "number", "minimum": -90, "maximum": 90 }, + "lon": { "type": "number", "minimum": -180, "maximum": 180 } + } + }, + "bbox": { + "type": "array", + "description": "[min_lon, min_lat, max_lon, max_lat]", + "items": { "type": "number" }, + "minItems": 4, + "maxItems": 4 + }, + "gaia_cells": { + "type": "array", + "description": "GAIA grounding — the environmental world-model cells this region is read through. Must be non-empty.", + "items": { "type": "string", "minLength": 1 }, + "minItems": 1 + } + } +} diff --git a/schemas/StateVector.json b/schemas/StateVector.json new file mode 100644 index 0000000..baeae1c --- /dev/null +++ b/schemas/StateVector.json @@ -0,0 +1,21 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://schemas.srcos.ai/v2/StateVector.json", + "title": "StateVector", + "description": "A reading of a twin's latent world-state x at a point in the replay sequence. Each named dimension is normalized to [0,1]; an out-of-range component is not a valid state and is rejected. A StateVector with no dimensions is not observable and is rejected. Provenance ties the reading back to the Impulse(s) that produced it.", + "type": "object", + "additionalProperties": false, + "required": ["type", "twin_id", "seq", "at", "dims", "provenance_refs"], + "properties": { + "type": { "const": "StateVector" }, + "twin_id": { "type": "string", "minLength": 1 }, + "seq": { "type": "integer", "minimum": 0 }, + "at": { "type": "string", "format": "date-time" }, + "dims": { + "type": "object", + "minProperties": 1, + "additionalProperties": { "type": "number", "minimum": 0, "maximum": 1 } + }, + "provenance_refs": { "type": "array", "items": { "type": "string" }, "minItems": 1 } + } +} diff --git a/schemas/WorldModel.json b/schemas/WorldModel.json new file mode 100644 index 0000000..81a7493 --- /dev/null +++ b/schemas/WorldModel.json @@ -0,0 +1,52 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://schemas.srcos.ai/v2/WorldModel.json", + "title": "WorldModel", + "description": "The state-space world model of a Twin — the dynamical layer beneath the governance lifecycle. It declares the state dimensions, the discrete dynamics x+ = A*x + B*(G*u) (A as a relaxation-toward-baseline rate), the impulse gates, the Region it is grounded in, and its exogenous source. GAIA is the exogenous world model: a WorldModel whose gates admit exogenous_shock MUST declare exogenous_source 'gaia' — you cannot admit exogenous shocks with no world model behind them (asserted by rejection). A model with no state dimensions or no gates is not a model and is rejected.", + "type": "object", + "additionalProperties": false, + "required": ["type", "model_id", "twin_id", "state_dims", "dynamics", "gates", "exogenous_source"], + "properties": { + "type": { "const": "WorldModel" }, + "model_id": { "type": "string", "minLength": 1 }, + "twin_id": { "type": "string", "minLength": 1 }, + "state_dims": { "type": "array", "items": { "type": "string", "minLength": 1 }, "minItems": 1 }, + "dynamics": { + "type": "object", + "additionalProperties": false, + "required": ["relaxation"], + "properties": { + "relaxation": { "type": "number", "minimum": 0, "maximum": 1 }, + "baseline": { "type": "object", "additionalProperties": { "type": "number", "minimum": 0, "maximum": 1 } } + } + }, + "gates": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "additionalProperties": false, + "required": ["impulse_class", "mode"], + "properties": { + "impulse_class": { "type": "string", "enum": ["device_reading", "market_data_event", "exogenous_shock", "policy_change", "human_action"] }, + "mode": { "type": "string", "enum": ["open", "attenuated", "closed"] } + } + } + }, + "region_ref": { "type": "string", "description": "Region.region_id this world model is grounded in." }, + "exogenous_source": { "type": "string", "enum": ["gaia", "none"] } + }, + "allOf": [ + { + "if": { + "properties": { + "gates": { "contains": { "type": "object", "properties": { "impulse_class": { "const": "exogenous_shock" } }, "required": ["impulse_class"] } } + } + }, + "then": { + "properties": { "exogenous_source": { "const": "gaia" } }, + "required": ["region_ref"] + } + } + ] +} diff --git a/tools/validate_eval_item_examples.py b/tools/validate_eval_item_examples.py new file mode 100644 index 0000000..b7a09fc --- /dev/null +++ b/tools/validate_eval_item_examples.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +"""Validate the EvalItem schema — the atomic contract of the Provider Eval Seed +Strategy that feeds the Intelligence-Superiority Bench (iSOTA). + +An EvalItem declares which corpus it belongs to (A provider-seed / B Sherlock-task / +C adversarial), its task family, the question, the traits an answer must have, how +it is graded, and its risk class. Load-bearing minimums are asserted by REJECTION: +an item with no expected traits is ungradeable, and a provider-sourced item must +name its provider (the harness stays neutral — provider is a label, never a winner). +""" + +from __future__ import annotations + +import copy +import json +import sys +from pathlib import Path + +from jsonschema import Draft202012Validator + +ROOT = Path(__file__).resolve().parents[1] +SCHEMA = ROOT / "schemas" / "EvalItem.json" +EXAMPLE = ROOT / "examples" / "eval-item.json" + + +def load(path: Path) -> dict: + return json.loads(path.read_text(encoding="utf-8")) + + +def fail(msg: str) -> None: + print(f"FAIL: {msg}") + sys.exit(1) + + +def main() -> int: + schema = load(SCHEMA) + Draft202012Validator.check_schema(schema) + v = Draft202012Validator(schema) + + example = load(EXAMPLE) + errors = sorted(v.iter_errors(example), key=lambda e: list(e.path)) + if errors: + loc = ".".join(str(p) for p in errors[0].path) or "" + fail(f"eval-item.json invalid: {loc}: {errors[0].message}") + + def must_reject(label: str, mutate) -> None: + doc = mutate(copy.deepcopy(example)) + if v.is_valid(doc): + fail(f"{label}: document was ACCEPTED but must be rejected") + + def drop(key): + return lambda d: {k: val for k, val in d.items() if k != key} + + must_reject("EvalItem empty expected_answer_traits (ungradeable)", lambda d: {**d, "expected_answer_traits": []}) + must_reject("EvalItem unknown task_family", lambda d: {**d, "task_family": "vibes_check"}) + must_reject("EvalItem unknown corpus", lambda d: {**d, "corpus": "D"}) + must_reject("EvalItem missing grading_method", drop("grading_method")) + must_reject("EvalItem missing risk_class", drop("risk_class")) + must_reject("EvalItem provider source without provider name", + lambda d: {**{k: val for k, val in d.items() if k != "provider"}, "source": "provider"}) + + print("OK: EvalItem — schema + example validated; 6 rejection invariants enforced") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/validate_world_model_examples.py b/tools/validate_world_model_examples.py new file mode 100644 index 0000000..6d63ead --- /dev/null +++ b/tools/validate_world_model_examples.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 +"""Validate the Twin World-Model & GAIA schema pack. + +Six canonical schemas give a Twin a dynamical, geospatially-grounded world model +beneath its governance lifecycle: WorldModel (x+ = A*x + B*(G*u)), StateVector +(the latent state reading), ImpulseGate (governed admission), Impulse (a typed +input), Region (geospatial grounding), and GaiaObservation (the exogenous state +read from GAIA — the estate's weather/environmental world model). + +The exit criterion — "schemas validate, example objects compile" — is enforced +here, and every load-bearing minimum is asserted by REJECTION. In particular the +two GAIA/fail-closed couplings are proven both ways: an exogenous shock with no +GAIA observation behind it is rejected, and a closed gate that still admits is +rejected. +""" + +from __future__ import annotations + +import copy +import json +import sys +from pathlib import Path + +from jsonschema import Draft202012Validator + +ROOT = Path(__file__).resolve().parents[1] +SCHEMAS = ROOT / "schemas" +EXAMPLES = ROOT / "examples" + +# schema file -> example file (both carry a `type` discriminator for the CI mapper) +PACK = { + "WorldModel.json": "world-model.json", + "StateVector.json": "state-vector.json", + "ImpulseGate.json": "impulse-gate.json", + "Impulse.json": "impulse.json", + "Region.json": "region.json", + "GaiaObservation.json": "gaia-observation.json", +} + + +def load(path: Path) -> dict: + return json.loads(path.read_text(encoding="utf-8")) + + +def fail(msg: str) -> None: + print(f"FAIL: {msg}") + sys.exit(1) + + +def main() -> int: + validators: dict[str, Draft202012Validator] = {} + for schema_file, example_file in PACK.items(): + schema = load(SCHEMAS / schema_file) + Draft202012Validator.check_schema(schema) + v = Draft202012Validator(schema) + validators[schema_file] = v + example = load(EXAMPLES / example_file) + errors = sorted(v.iter_errors(example), key=lambda e: list(e.path)) + if errors: + loc = ".".join(str(p) for p in errors[0].path) or "" + fail(f"{example_file} invalid against {schema_file}: {loc}: {errors[0].message}") + + def must_reject(schema_file: str, label: str, mutate) -> None: + base = load(EXAMPLES / PACK[schema_file]) + doc = mutate(copy.deepcopy(base)) + if validators[schema_file].is_valid(doc): + fail(f"{label}: document was ACCEPTED but must be rejected") + + def drop(key): + return lambda d: {k: v for k, v in d.items() if k != key} + + # GaiaObservation: an ungrounded / unprovenanced / mis-typed reading is inadmissible. + must_reject("GaiaObservation.json", "GaiaObservation empty provenance_refs", lambda d: {**d, "provenance_refs": []}) + must_reject("GaiaObservation.json", "GaiaObservation missing gaia_cell", drop("gaia_cell")) + must_reject("GaiaObservation.json", "GaiaObservation unknown field", lambda d: {**d, "field": "vibes"}) + + # Region: a region with no GAIA cells cannot supply exogenous state. + must_reject("Region.json", "Region empty gaia_cells", lambda d: {**d, "gaia_cells": []}) + must_reject("Region.json", "Region missing centroid", drop("centroid")) + + # ImpulseGate: fail-closed means a closed gate cannot pass signal. + must_reject("ImpulseGate.json", "ImpulseGate closed but gate_factor>0", + lambda d: {**d, "mode": "closed", "gate_factor": 0.5}) + must_reject("ImpulseGate.json", "ImpulseGate unknown impulse_class", lambda d: {**d, "impulse_class": "telepathy"}) + + # Impulse: exogenous shocks MUST cite GAIA; a closed gate cannot admit. + must_reject("Impulse.json", "Impulse exogenous_shock without GAIA source_ref", drop("source_ref")) + must_reject("Impulse.json", "Impulse admitted through closed gate", + lambda d: {**d, "gate_state": "closed", "admitted": True}) + + # StateVector: no dimensions = unobservable; a component outside [0,1] is not a state. + must_reject("StateVector.json", "StateVector empty dims", lambda d: {**d, "dims": {}}) + must_reject("StateVector.json", "StateVector dim out of range", lambda d: {**d, "dims": {"risk": 1.5}}) + must_reject("StateVector.json", "StateVector missing twin_id", drop("twin_id")) + + # WorldModel: no dims / no gates = not a model; exogenous gate without GAIA source is rejected. + must_reject("WorldModel.json", "WorldModel empty state_dims", lambda d: {**d, "state_dims": []}) + must_reject("WorldModel.json", "WorldModel empty gates", lambda d: {**d, "gates": []}) + must_reject("WorldModel.json", "WorldModel exogenous gate but exogenous_source=none", + lambda d: {**d, "exogenous_source": "none"}) + must_reject("WorldModel.json", "WorldModel missing twin_id", drop("twin_id")) + + print( + "OK: world-model + GAIA pack — 6 schemas + examples validated " + "(WorldModel, StateVector, ImpulseGate, Impulse, Region, GaiaObservation); " + "16 rejection invariants enforced (incl. exogenous-shock⇒GAIA and closed-gate⇒no-admit)" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())