From f15efc469cffeb7db866456400e641f6b312b13b Mon Sep 17 00:00:00 2001 From: Michael Heller <21163552+mdheller@users.noreply.github.com> Date: Sun, 2 Aug 2026 16:00:57 -0400 Subject: [PATCH 1/4] =?UTF-8?q?feat(data):=20DataClass=20(OntoDT/OntoDQ)?= =?UTF-8?q?=20+=20TF-Lattice=20classifier=20=E2=80=94=20data-side=20govern?= =?UTF-8?q?ance=20(task=20#14)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The data-side counterpart to the glossary (biz-side, #250). A DataClass binds, fail-closed: OntoDT datatype-ontology class (capture) + business GlossaryTerm (biz↔data) + ValidValues domain (enum/range/regex) + an optional TF-Lattice wide-and-deep classifier that ASSIGNS the class by inference (OntoDQ) and is a first-class CATALOGED model — modelRef→ModelManifest, runRef→RunRecord, compute on Ray/TritFabric, monotonicFeatures lattice constraint, labels that are GlossaryTerm URNs (assigned in the glossary). EntityField gains an optional dataClassRef. Drift-guard (validate_data_class_examples.py, -resolved via a referencing registry): ontological typing + glossary link + domain required; classifier labels are glossary terms, model is cataloged, compute is ray/tritfabric; a field bound to a DataClass must share its domain kind. 4 negative vectors; teeth-verified (bad-label, kind-mismatch). make validate ok; dup-$id 346. Ties biz-glossary #250 ↔ DataClass ↔ EntityField ↔ model catalog ↔ Ray/TritFabric runs. --- CHANGELOG.md | 1 + Makefile | 8 +- examples/data_class.currency.json | 37 ++++++ examples/entity_field.revenue.json | 16 +++ fixtures/data-class/conformance.json | 169 ++++++++++++++++++++++++++ schemas/DataClass.json | 133 ++++++++++++++++++++ schemas/EntityField.json | 8 ++ specs/data-class-contract.md | 23 ++++ tools/validate_data_class_examples.py | 135 ++++++++++++++++++++ 9 files changed, 528 insertions(+), 2 deletions(-) create mode 100644 examples/data_class.currency.json create mode 100644 examples/entity_field.revenue.json create mode 100644 fixtures/data-class/conformance.json create mode 100644 schemas/DataClass.json create mode 100644 specs/data-class-contract.md create mode 100644 tools/validate_data_class_examples.py diff --git a/CHANGELOG.md b/CHANGELOG.md index cf8ee6f..d411188 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ The format follows [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). Thi ## [Unreleased] ### Added +- DataClass v0.1 (OntoDT/OntoDQ data-side governance, task #14; AMG 2.0 deck): a `DataClass` binds a datatype-ontology class (OntoDT), the business `GlossaryTerm` (biz↔data), and a `ValidValues` domain (enum/range/regex), with an optional **TF-Lattice wide-and-deep** classifier that is a first-class CATALOGED model — `modelRef`→`ModelManifest`, `runRef`→`RunRecord`, `compute` on **Ray/TritFabric**, `monotonicFeatures` lattice constraint, and `labels` that are `GlossaryTerm` URNs (assigned in the glossary). `EntityField` gains an optional `dataClassRef`; the drift-guard enforces field↔class domain conformance + classifier integrity (labels are glossary terms, model is cataloged, compute is ray/tritfabric) + ontological typing. 4 negative vectors, teeth-verified. Backward-compatible. - Glossary alignment v0.1 (vocabulary as governance substrate, task #13): `GlossaryTerm` extended with `partOfSpeech` (NP/VP, Tesnière), `status`, a 3-method `alignment` (formal ontology class + NP↔VP vector link in the sovereign 768 space + estate entity/service binding), and typed `relations` over a closed RDF/RDFS/SKOS/FOAF predicate set (is-type/has-type, is-a, has-a, is-member/has-member, skos:broader/narrower/related, foaf:member, has-datatype). Fail-closed drift-guard (`validate_glossary_alignment_examples.py`): an `approved` term must be captured+vector-aligned+implemented (NP→entity, VP→action/service) or it's a governance hole; vector links must be reciprocal; relation inverse-pairs reciprocate; `skos:related` is symmetric; `is-a` subsumption is acyclic. Teeth-verified. Backward-compatible (optional). Follow-ups: LSA↔LDA currency detection + agreement test vs the blast-radius graph + neurosymbolic domain. - Receipt **epistemicLevel** (SP-GATE-003, closes audit G3): `ReleaseReceipt` and `SyncCycleReceipt` may now carry `epistemicLevel` (`Proved`/`Speculative`/`Refuted`) + an `attestation` block — the MEET of attestation ∧ verification. Optional + backward-compatible; `tools/validate_sync_cycle_receipts.py` RECOMPUTES the meet and rejects a declared value that disagrees, so a receipt can never claim `Proved` while unsigned or after a failed check (absence of a signature returns Speculative, not Proved). Consumes the explicit `attestation.signed` from SP-GATE-002 (source-os #312) and the SyncCycleReceipt attestation from SP-GATE-001. Examples: verified+signed→Proved, verified-but-unsigned→Speculative, failed→Refuted, dry-run→Speculative. Teeth-verified. - Data-Acquisition **citable commons** (WO-COMMONS-2): a promoted platform `CatalogEntry` may carry `promotion.doi` and the approving `DataAcquisitionRequest` records the same `mintedDoi` — a citable DOI (Zenodo/DataCite-style) minted when the governance ∧ IP/legal meet approves promotion. Invariants (schema `allOf` + validator): a DOI may appear only on a `promoted` entry, `mintedDoi` only on an `approved` DAR, and where both are present they must be equal (minted once). Grounds the governed platform tier as a citable open commons. diff --git a/Makefile b/Makefile index 46fc379..4749a7e 100644 --- a/Makefile +++ b/Makefile @@ -1,12 +1,16 @@ -.PHONY: validate validate-resource-contract validate-measurement validate-value-type validate-source-locator validate-sourceos-repo-manifest validate-mesh-action-registry validate-control-plane-examples validate-nlboot-examples validate-lattice-data-governai-examples validate-ops-history-examples validate-runtime-observability-examples validate-interpretability-examples validate-lifecycle-boundary-examples validate-svf-contracts validate-sync-cycle-receipts validate-onboarding-examples validate-runtime-causality-examples validate-agentic-os-examples validate-triparty-examples validate-labor-market-examples validate-supply-chain-risk-examples validate-reasoning-examples validate-mpcc-event-examples validate-knowledge-nugget-examples validate-semantic-action-examples validate-epistemic-kernel-examples validate-ab-update-examples validate-device-service-examples validate-duplicate-schema-ids validate-lawful-dispatch-receipt validate-architectural-building-block validate-agent-passport-examples validate-seam-definition-examples validate-agent-system-vocabulary validate-genesis-inception-examples validate-measurement validate-world-model-examples validate-eval-item-examples validate-ingestion-pipeline-examples validate-data-acquisition-examples validate-glossary-alignment-examples +.PHONY: validate validate-resource-contract validate-measurement validate-value-type validate-source-locator validate-sourceos-repo-manifest validate-mesh-action-registry validate-control-plane-examples validate-nlboot-examples validate-lattice-data-governai-examples validate-ops-history-examples validate-runtime-observability-examples validate-interpretability-examples validate-lifecycle-boundary-examples validate-svf-contracts validate-sync-cycle-receipts validate-onboarding-examples validate-runtime-causality-examples validate-agentic-os-examples validate-triparty-examples validate-labor-market-examples validate-supply-chain-risk-examples validate-reasoning-examples validate-mpcc-event-examples validate-knowledge-nugget-examples validate-semantic-action-examples validate-epistemic-kernel-examples validate-ab-update-examples validate-device-service-examples validate-duplicate-schema-ids validate-lawful-dispatch-receipt validate-architectural-building-block validate-agent-passport-examples validate-seam-definition-examples validate-agent-system-vocabulary validate-genesis-inception-examples validate-measurement validate-world-model-examples validate-eval-item-examples validate-ingestion-pipeline-examples validate-data-acquisition-examples validate-glossary-alignment-examples validate-data-class-examples -validate: validate-glossary-alignment-examples validate-data-acquisition-examples validate-ingestion-pipeline-examples validate-control-plane-examples validate-nlboot-examples validate-lattice-data-governai-examples validate-ops-history-examples validate-runtime-observability-examples validate-interpretability-examples validate-lifecycle-boundary-examples validate-svf-contracts validate-sync-cycle-receipts validate-onboarding-examples validate-runtime-causality-examples validate-agentic-os-examples validate-triparty-examples validate-labor-market-examples validate-supply-chain-risk-examples validate-reasoning-examples validate-mpcc-event-examples validate-knowledge-nugget-examples validate-semantic-action-examples validate-epistemic-kernel-examples validate-ab-update-examples validate-device-service-examples validate-duplicate-schema-ids validate-value-type validate-source-locator validate-sourceos-repo-manifest validate-mesh-action-registry validate-lawful-dispatch-receipt validate-architectural-building-block validate-agent-passport-examples validate-seam-definition-examples validate-agent-system-vocabulary validate-genesis-inception-examples validate-measurement validate-world-model-examples validate-eval-item-examples validate-resource-contract +validate: validate-data-class-examples validate-glossary-alignment-examples validate-data-acquisition-examples validate-ingestion-pipeline-examples validate-control-plane-examples validate-nlboot-examples validate-lattice-data-governai-examples validate-ops-history-examples validate-runtime-observability-examples validate-interpretability-examples validate-lifecycle-boundary-examples validate-svf-contracts validate-sync-cycle-receipts validate-onboarding-examples validate-runtime-causality-examples validate-agentic-os-examples validate-triparty-examples validate-labor-market-examples validate-supply-chain-risk-examples validate-reasoning-examples validate-mpcc-event-examples validate-knowledge-nugget-examples validate-semantic-action-examples validate-epistemic-kernel-examples validate-ab-update-examples validate-device-service-examples validate-duplicate-schema-ids validate-value-type validate-source-locator validate-sourceos-repo-manifest validate-mesh-action-registry validate-lawful-dispatch-receipt validate-architectural-building-block validate-agent-passport-examples validate-seam-definition-examples validate-agent-system-vocabulary validate-genesis-inception-examples validate-measurement validate-world-model-examples validate-eval-item-examples validate-resource-contract @echo "OK: validate" validate-ingestion-pipeline-examples: python3 -m pip install --user jsonschema >/dev/null python3 tools/validate_ingestion_pipeline_examples.py +validate-data-class-examples: + python3 -m pip install --user jsonschema >/dev/null + python3 tools/validate_data_class_examples.py + validate-glossary-alignment-examples: python3 -m pip install --user jsonschema >/dev/null python3 tools/validate_glossary_alignment_examples.py diff --git a/examples/data_class.currency.json b/examples/data_class.currency.json new file mode 100644 index 0000000..5da6589 --- /dev/null +++ b/examples/data_class.currency.json @@ -0,0 +1,37 @@ +{ + "id": "urn:srcos:data-class:currency-amount-usd", + "type": "DataClass", + "specVersion": "2.0.0", + "ontologyClassRef": "urn:srcos:ontology-class:datatype.CurrencyAmount", + "glossaryTermRef": "urn:srcos:glossary:revenue", + "domain": { + "kind": "range", + "values": [], + "min": 0, + "max": null, + "pattern": null + }, + "dataQualityRules": [ + "dq.nonnegative", + "dq.currency.usd.scale2" + ], + "classifier": { + "kind": "tf-lattice-wide-and-deep", + "modelRef": "urn:srcos:model-manifest:dataclass-assigner-lattice-v1", + "version": "1.0.0", + "runRef": "urn:srcos:run:dataclass-assigner-train-001", + "compute": { + "platform": "ray", + "assetRef": "urn:srcos:physical-asset:ray-cluster-01", + "serviceRef": "urn:srcos:tritfabric-consumption-api:assigner" + }, + "monotonicFeatures": [ + "token_magnitude", + "decimal_places" + ], + "labels": [ + "urn:srcos:glossary:revenue", + "urn:srcos:glossary:cost" + ] + } +} diff --git a/examples/entity_field.revenue.json b/examples/entity_field.revenue.json new file mode 100644 index 0000000..2b9ad95 --- /dev/null +++ b/examples/entity_field.revenue.json @@ -0,0 +1,16 @@ +{ + "name": "revenue_usd", + "dataType": "decimal", + "nullable": false, + "validValues": { + "kind": "range", + "values": [], + "min": 0, + "max": null, + "pattern": null + }, + "glossaryTerms": [ + "urn:srcos:glossary:revenue" + ], + "dataClassRef": "urn:srcos:data-class:currency-amount-usd" +} diff --git a/fixtures/data-class/conformance.json b/fixtures/data-class/conformance.json new file mode 100644 index 0000000..bae97de --- /dev/null +++ b/fixtures/data-class/conformance.json @@ -0,0 +1,169 @@ +{ + "description": "Negative vectors for the DataClass (OntoDT/OntoDQ) family v0.1. Each MUST fail on its named failValidator; validate_data_class_examples.py enforces both, plus: a DataClass must be ontologically typed + glossary-linked + domain-bounded; a classifier's labels are GlossaryTerms and its compute is Ray/TritFabric; and a bound EntityField's validValues must conform to the DataClass domain.", + "cases": [ + { + "schema": "DataClass.json", + "reason": "classifier kind is pinned to tf-lattice-wide-and-deep.", + "failValidator": "const", + "document": { + "id": "urn:srcos:data-class:currency-amount-usd", + "type": "DataClass", + "specVersion": "2.0.0", + "ontologyClassRef": "urn:srcos:ontology-class:datatype.CurrencyAmount", + "glossaryTermRef": "urn:srcos:glossary:revenue", + "domain": { + "kind": "range", + "values": [], + "min": 0, + "max": null, + "pattern": null + }, + "dataQualityRules": [ + "dq.nonnegative", + "dq.currency.usd.scale2" + ], + "classifier": { + "kind": "random-forest", + "modelRef": "urn:srcos:model-manifest:dataclass-assigner-lattice-v1", + "version": "1.0.0", + "runRef": "urn:srcos:run:dataclass-assigner-train-001", + "compute": { + "platform": "ray", + "assetRef": "urn:srcos:physical-asset:ray-cluster-01", + "serviceRef": "urn:srcos:tritfabric-consumption-api:assigner" + }, + "monotonicFeatures": [ + "token_magnitude", + "decimal_places" + ], + "labels": [ + "urn:srcos:glossary:revenue", + "urn:srcos:glossary:cost" + ] + } + } + }, + { + "schema": "DataClass.json", + "reason": "classifier compute platform is closed (ray|tritfabric).", + "failValidator": "enum", + "document": { + "id": "urn:srcos:data-class:currency-amount-usd", + "type": "DataClass", + "specVersion": "2.0.0", + "ontologyClassRef": "urn:srcos:ontology-class:datatype.CurrencyAmount", + "glossaryTermRef": "urn:srcos:glossary:revenue", + "domain": { + "kind": "range", + "values": [], + "min": 0, + "max": null, + "pattern": null + }, + "dataQualityRules": [ + "dq.nonnegative", + "dq.currency.usd.scale2" + ], + "classifier": { + "kind": "tf-lattice-wide-and-deep", + "modelRef": "urn:srcos:model-manifest:dataclass-assigner-lattice-v1", + "version": "1.0.0", + "runRef": "urn:srcos:run:dataclass-assigner-train-001", + "compute": { + "platform": "lambda" + }, + "monotonicFeatures": [ + "token_magnitude", + "decimal_places" + ], + "labels": [ + "urn:srcos:glossary:revenue", + "urn:srcos:glossary:cost" + ] + } + } + }, + { + "schema": "DataClass.json", + "reason": "classifier labels must be GlossaryTerm URNs (assigned in the glossary).", + "failValidator": "pattern", + "document": { + "id": "urn:srcos:data-class:currency-amount-usd", + "type": "DataClass", + "specVersion": "2.0.0", + "ontologyClassRef": "urn:srcos:ontology-class:datatype.CurrencyAmount", + "glossaryTermRef": "urn:srcos:glossary:revenue", + "domain": { + "kind": "range", + "values": [], + "min": 0, + "max": null, + "pattern": null + }, + "dataQualityRules": [ + "dq.nonnegative", + "dq.currency.usd.scale2" + ], + "classifier": { + "kind": "tf-lattice-wide-and-deep", + "modelRef": "urn:srcos:model-manifest:dataclass-assigner-lattice-v1", + "version": "1.0.0", + "runRef": "urn:srcos:run:dataclass-assigner-train-001", + "compute": { + "platform": "ray", + "assetRef": "urn:srcos:physical-asset:ray-cluster-01", + "serviceRef": "urn:srcos:tritfabric-consumption-api:assigner" + }, + "monotonicFeatures": [ + "token_magnitude", + "decimal_places" + ], + "labels": [ + "not-a-glossary-urn" + ] + } + } + }, + { + "schema": "DataClass.json", + "reason": "ontologyClassRef is required — a data class must be ontologically typed.", + "failValidator": "required", + "document": { + "id": "urn:srcos:data-class:currency-amount-usd", + "type": "DataClass", + "specVersion": "2.0.0", + "glossaryTermRef": "urn:srcos:glossary:revenue", + "domain": { + "kind": "range", + "values": [], + "min": 0, + "max": null, + "pattern": null + }, + "dataQualityRules": [ + "dq.nonnegative", + "dq.currency.usd.scale2" + ], + "classifier": { + "kind": "tf-lattice-wide-and-deep", + "modelRef": "urn:srcos:model-manifest:dataclass-assigner-lattice-v1", + "version": "1.0.0", + "runRef": "urn:srcos:run:dataclass-assigner-train-001", + "compute": { + "platform": "ray", + "assetRef": "urn:srcos:physical-asset:ray-cluster-01", + "serviceRef": "urn:srcos:tritfabric-consumption-api:assigner" + }, + "monotonicFeatures": [ + "token_magnitude", + "decimal_places" + ], + "labels": [ + "urn:srcos:glossary:revenue", + "urn:srcos:glossary:cost" + ] + } + } + } + ] +} diff --git a/schemas/DataClass.json b/schemas/DataClass.json new file mode 100644 index 0000000..0df15cc --- /dev/null +++ b/schemas/DataClass.json @@ -0,0 +1,133 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://schemas.srcos.ai/v2/DataClass.json", + "title": "DataClass", + "description": "An OntoDT/OntoDQ data class: a data type/class defined ontologically (mereotopology in the datatype ontology) with its value domain, its business glossary term, and an optional TF-Lattice wide-and-deep classifier — registered in the model catalog with a run on Ray/TritFabric — that assigns this class by inference. Bridges the business vocabulary (GlossaryTerm) to the data (EntityField). Governs data quality: a field bound to this class must conform to its domain.", + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "type", + "specVersion", + "ontologyClassRef", + "glossaryTermRef", + "domain" + ], + "properties": { + "id": { + "type": "string", + "pattern": "^urn:srcos:data-class:[A-Za-z0-9._~:-]+$" + }, + "type": { + "const": "DataClass" + }, + "specVersion": { + "type": "string" + }, + "ontologyClassRef": { + "type": "string", + "pattern": "^urn:srcos:ontology-class:[A-Za-z0-9._~:-]+$", + "description": "OntoDT — the datatype-ontology class this data class IS (capture; enables inference-based data-quality logic)." + }, + "glossaryTermRef": { + "type": "string", + "pattern": "^urn:srcos:glossary:", + "description": "The business GlossaryTerm this data class realizes — the biz↔data bridge." + }, + "domain": { + "$ref": "ValidValues.json", + "description": "The value domain: enumeration / range / regex. Reuses ValidValues (single source with EntityField.validValues)." + }, + "dataQualityRules": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Identifiers of data-quality validations enforced for this class (OntoDQ)." + }, + "classifier": { + "type": [ + "object", + "null" + ], + "additionalProperties": false, + "description": "Optional TF-Lattice wide-and-deep classifier that ASSIGNS this data class by inference. It is a first-class cataloged model: modelRef → ModelManifest, runRef → RunRecord, compute on Ray/TritFabric, labels are GlossaryTerms.", + "required": [ + "kind", + "modelRef", + "version", + "labels" + ], + "properties": { + "kind": { + "const": "tf-lattice-wide-and-deep" + }, + "modelRef": { + "type": "string", + "pattern": "^urn:srcos:model-manifest:", + "description": "The classifier model in the model catalog (ModelManifest)." + }, + "version": { + "type": "string" + }, + "runRef": { + "type": [ + "string", + "null" + ], + "pattern": "^urn:srcos:run:", + "description": "The training/eval RunRecord." + }, + "compute": { + "type": [ + "object", + "null" + ], + "additionalProperties": false, + "required": [ + "platform" + ], + "properties": { + "platform": { + "type": "string", + "enum": [ + "ray", + "tritfabric" + ] + }, + "assetRef": { + "type": [ + "string", + "null" + ], + "pattern": "^urn:srcos:[A-Za-z0-9._~:-]+$" + }, + "serviceRef": { + "type": [ + "string", + "null" + ], + "pattern": "^urn:srcos:[A-Za-z0-9._~:-]+$" + } + } + }, + "monotonicFeatures": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Features constrained monotone in the lattice (TF-Lattice shape constraint)." + }, + "labels": { + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "pattern": "^urn:srcos:glossary:" + }, + "description": "Output class labels — each a GlossaryTerm URN (labels are assigned in the glossary)." + } + } + } + } +} diff --git a/schemas/EntityField.json b/schemas/EntityField.json index 62abddb..217c347 100644 --- a/schemas/EntityField.json +++ b/schemas/EntityField.json @@ -45,6 +45,14 @@ "pattern": "^urn:srcos:glossary:" }, "description": "URNs of GlossaryTerms that this field represents. Pattern: urn:srcos:glossary:" + }, + "dataClassRef": { + "type": [ + "string", + "null" + ], + "pattern": "^urn:srcos:data-class:[A-Za-z0-9._~:-]+$", + "description": "Optional binding to the OntoDQ DataClass this field is an instance of. When set, the field's validValues must conform to the DataClass domain (validator)." } } } diff --git a/specs/data-class-contract.md b/specs/data-class-contract.md new file mode 100644 index 0000000..1faa834 --- /dev/null +++ b/specs/data-class-contract.md @@ -0,0 +1,23 @@ +# DataClass Contract (v0.1) — OntoDT / OntoDQ, the data side of governance + +The business vocabulary (`GlossaryTerm`) says what a thing MEANS; the `DataClass` says what +its DATA is, ontologically — so data quality can be inferred and enforced, not asserted. + +A `DataClass` binds four things and is fail-closed if any is missing: +- **OntoDT** — `ontologyClassRef`: the datatype-ontology class (capture; enables reusable, + inferable data-quality logic — IBM OntoDT). +- **biz↔data** — `glossaryTermRef`: the business `GlossaryTerm` this data class realizes. +- **domain** — `domain` ($ref `ValidValues`): enumeration / range / regex (valid values, + bounds, pattern), the same constraint an `EntityField` carries. +- **assignment** (optional) — a **TF-Lattice wide-and-deep** `classifier` that assigns this + class by inference (OntoDQ). It is a first-class **cataloged model**: `modelRef` → + `ModelManifest`, `runRef` → `RunRecord`, `compute.platform` ∈ {ray, tritfabric} with an + asset/service ref, `monotonicFeatures` for the lattice shape constraint, and `labels` that + are **`GlossaryTerm` URNs** (labels are assigned in the glossary). + +`EntityField` gains an optional `dataClassRef`. The validator enforces field↔class conformance: +a field bound to a DataClass must share its `domain.kind` — a field cannot claim a class it +does not fit. + +## Conformance +`make validate-data-class-examples` (in `make validate`). diff --git a/tools/validate_data_class_examples.py b/tools/validate_data_class_examples.py new file mode 100644 index 0000000..64201d0 --- /dev/null +++ b/tools/validate_data_class_examples.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 +"""Validate the DataClass (OntoDT/OntoDQ) family + its cross-bindings (task #14). + +A DataClass is the data-side governance node: ontologically typed (OntoDT), glossary-linked +(biz↔data), domain-bounded (ValidValues), optionally assigned by a TF-Lattice classifier that +is a CATALOGED model (ModelManifest) with a run on Ray/TritFabric and glossary-term labels. + +Checks (fail-closed): + 1. schema conformance (with ValidValues $ref resolved); + 2. classifier integrity — kind pinned, modelRef→model-manifest, runRef→run, compute on + ray|tritfabric, and every label a GlossaryTerm URN (labels live in the glossary); + 3. field↔class domain conformance — an EntityField.dataClassRef binding a DataClass in the + set must share the DataClass domain 'kind' (a field can't claim a class it doesn't fit); + 4. negative vectors fail on their named keyword. +""" +from __future__ import annotations + +import json +import sys +from pathlib import Path + +import jsonschema +from referencing import Registry, Resource + +ROOT = Path(__file__).resolve().parents[1] +SCHEMAS = ROOT / "schemas" + +FAILURES: list[str] = [] +CHECKS: dict[str, bool] = {} + + +def load(p: Path) -> dict: + return json.loads(p.read_text(encoding="utf-8")) + + +def registry() -> Registry: + # Resolve local $refs (e.g. "ValidValues.json") by loading every schema under its $id + # AND under its bare filename, so relative refs resolve without a network fetch. + resources = [] + for f in SCHEMAS.glob("*.json"): + schema = load(f) + res = Resource.from_contents(schema) + resources.append((schema.get("$id", f.name), res)) + resources.append((f.name, res)) + return Registry().with_resources(resources) + + +REG = registry() + + +def validator_for(schema: dict) -> jsonschema.Draft202012Validator: + return jsonschema.Draft202012Validator(schema, registry=REG) + + +def check_conformance(dataclass_schema, field_schema, dcs, fields) -> None: + for name, d in {**dcs, **fields}.items(): + schema = dataclass_schema if d.get("type") == "DataClass" else field_schema + errs = sorted(validator_for(schema).iter_errors(d), key=str) + if errs: + for e in errs: + FAILURES.append(f"{name}: {e.message}") + else: + CHECKS[f"schema:{name}"] = True + + +def check_classifier(dcs) -> None: + for name, dc in dcs.items(): + clf = dc.get("classifier") + if not clf: + CHECKS[f"classifier:{name}:none"] = True + continue + labels = clf.get("labels") or [] + if not all(l.startswith("urn:srcos:glossary:") for l in labels): + FAILURES.append(f"{name}: classifier labels must all be GlossaryTerm URNs (assigned in the glossary)") + elif not clf.get("modelRef", "").startswith("urn:srcos:model-manifest:"): + FAILURES.append(f"{name}: classifier.modelRef must be a ModelManifest (cataloged model)") + elif (clf.get("compute") or {}).get("platform") not in ("ray", "tritfabric"): + FAILURES.append(f"{name}: classifier compute must run on ray|tritfabric") + else: + CHECKS[f"classifier:{name}:cataloged-on-compute-with-glossary-labels"] = True + + +def check_field_conformance(dcs, fields) -> None: + by_id = {dc["id"]: dc for dc in dcs.values()} + for name, fld in fields.items(): + ref = fld.get("dataClassRef") + if not ref: + continue + dc = by_id.get(ref) + if dc is None: + continue # DataClass defined elsewhere + field_kind = (fld.get("validValues") or {}).get("kind") + class_kind = (dc.get("domain") or {}).get("kind") + if field_kind and class_kind and field_kind != class_kind: + FAILURES.append(f"{name}: validValues.kind {field_kind!r} does not conform to DataClass {ref} " + f"domain kind {class_kind!r}") + else: + CHECKS[f"field-conforms:{name}"] = True + + +def check_negatives(dataclass_schema) -> None: + fx = load(ROOT / "fixtures" / "data-class" / "conformance.json") + for i, case in enumerate(fx["cases"]): + expected = case.get("failValidator") + try: + validator_for(dataclass_schema).validate(case["document"]) + except jsonschema.ValidationError as exc: + if expected is not None and exc.validator != expected: + FAILURES.append(f"negative {i}: failed on {exc.validator!r}, not {expected!r}: {case['reason']}") + else: + CHECKS[f"negative:{i}:{exc.validator}"] = True + continue + FAILURES.append(f"negative {i} unexpectedly PASSED: {case['reason']}") + + +def main() -> int: + dataclass_schema = load(SCHEMAS / "DataClass.json") + field_schema = load(SCHEMAS / "EntityField.json") + dcs = {"data_class.currency.json": load(ROOT / "examples" / "data_class.currency.json")} + fields = {"entity_field.revenue.json": load(ROOT / "examples" / "entity_field.revenue.json")} + + check_conformance(dataclass_schema, field_schema, dcs, fields) + check_classifier(dcs) + check_field_conformance(dcs, fields) + check_negatives(dataclass_schema) + + for m in FAILURES: + print(f"FAIL: {m}", file=sys.stderr) + ok = not FAILURES and all(CHECKS.values()) + print(json.dumps({"ok": ok, "checks": CHECKS}, indent=2, sort_keys=True)) + return 0 if ok else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) From d25fa5cfdaa67cb70f215d6ec09a8322fe8fea65 Mon Sep 17 00:00:00 2001 From: Michael Heller <21163552+mdheller@users.noreply.github.com> Date: Sun, 2 Aug 2026 16:09:04 -0400 Subject: [PATCH 2/4] =?UTF-8?q?feat(data):=20two-level=20classifier=20?= =?UTF-8?q?=E2=80=94=20per-class=20LOGISTIC=20+=20per-table=20SOFTMAX=20(L?= =?UTF-8?q?SA=20+=20doc2vec)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per Charles: each class needs an individually-testable classifier, and each table needs a softmax aligning a LSA bag-of-words embedding and a doc2vec sentence-encoder (the n-ary logit→class examples). - DataClass.classifier is now the PER-CLASS head: head=logistic (one-vs-rest, binary) + a REQUIRED evalRunRef (the individual per-class test) + threshold. A class with no eval run is untestable and refused. - New TableClassifier: per-table head=softmax (n-ary), embeddings MUST include BOTH lsa-bag-of-words AND doc2vec-sentence-encoder, a cataloged ModelManifest with a run on Ray/TritFabric, assignsClasses = the N DataClasses. - Validator enforces both + 2 new negative vectors. Teeth-verified (missing-doc2vec, missing-evalRunRef). make validate ok; dup-$id 347. --- CHANGELOG.md | 2 +- examples/data_class.currency.json | 5 +- examples/table_classifier.finance.json | 27 ++++++ fixtures/data-class/conformance.json | 82 ++++++++++++++++- schemas/DataClass.json | 20 ++++ schemas/TableClassifier.json | 121 +++++++++++++++++++++++++ specs/data-class-contract.md | 14 +++ tools/validate_data_class_examples.py | 50 +++++++++- 8 files changed, 314 insertions(+), 7 deletions(-) create mode 100644 examples/table_classifier.finance.json create mode 100644 schemas/TableClassifier.json diff --git a/CHANGELOG.md b/CHANGELOG.md index d411188..f0d3c01 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ The format follows [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). Thi ## [Unreleased] ### Added -- DataClass v0.1 (OntoDT/OntoDQ data-side governance, task #14; AMG 2.0 deck): a `DataClass` binds a datatype-ontology class (OntoDT), the business `GlossaryTerm` (biz↔data), and a `ValidValues` domain (enum/range/regex), with an optional **TF-Lattice wide-and-deep** classifier that is a first-class CATALOGED model — `modelRef`→`ModelManifest`, `runRef`→`RunRecord`, `compute` on **Ray/TritFabric**, `monotonicFeatures` lattice constraint, and `labels` that are `GlossaryTerm` URNs (assigned in the glossary). `EntityField` gains an optional `dataClassRef`; the drift-guard enforces field↔class domain conformance + classifier integrity (labels are glossary terms, model is cataloged, compute is ray/tritfabric) + ontological typing. 4 negative vectors, teeth-verified. Backward-compatible. +- DataClass v0.1 (OntoDT/OntoDQ data-side governance, task #14; AMG 2.0 deck): a `DataClass` binds a datatype-ontology class (OntoDT), the business `GlossaryTerm` (biz↔data), and a `ValidValues` domain (enum/range/regex), with an optional **TF-Lattice wide-and-deep** classifier that is a first-class CATALOGED model — `modelRef`→`ModelManifest`, `runRef`→`RunRecord`, `compute` on **Ray/TritFabric**, `monotonicFeatures` lattice constraint, and `labels` that are `GlossaryTerm` URNs (assigned in the glossary). `EntityField` gains an optional `dataClassRef`; the drift-guard enforces field↔class domain conformance + classifier integrity (labels are glossary terms, model is cataloged, compute is ray/tritfabric) + ontological typing. 4 negative vectors, teeth-verified. Backward-compatible. Two-level classifier: per-class LOGISTIC head (`DataClass.classifier`, one-vs-rest, required `evalRunRef` — individually testable per class/glossary term) + a per-table SOFTMAX `TableClassifier` aligning an LSA bag-of-words embedding AND a doc2vec sentence-encoder (the n-ary logit→class assignment), a cataloged model on Ray/TritFabric assigning N DataClasses. Drift-guard enforces both. - Glossary alignment v0.1 (vocabulary as governance substrate, task #13): `GlossaryTerm` extended with `partOfSpeech` (NP/VP, Tesnière), `status`, a 3-method `alignment` (formal ontology class + NP↔VP vector link in the sovereign 768 space + estate entity/service binding), and typed `relations` over a closed RDF/RDFS/SKOS/FOAF predicate set (is-type/has-type, is-a, has-a, is-member/has-member, skos:broader/narrower/related, foaf:member, has-datatype). Fail-closed drift-guard (`validate_glossary_alignment_examples.py`): an `approved` term must be captured+vector-aligned+implemented (NP→entity, VP→action/service) or it's a governance hole; vector links must be reciprocal; relation inverse-pairs reciprocate; `skos:related` is symmetric; `is-a` subsumption is acyclic. Teeth-verified. Backward-compatible (optional). Follow-ups: LSA↔LDA currency detection + agreement test vs the blast-radius graph + neurosymbolic domain. - Receipt **epistemicLevel** (SP-GATE-003, closes audit G3): `ReleaseReceipt` and `SyncCycleReceipt` may now carry `epistemicLevel` (`Proved`/`Speculative`/`Refuted`) + an `attestation` block — the MEET of attestation ∧ verification. Optional + backward-compatible; `tools/validate_sync_cycle_receipts.py` RECOMPUTES the meet and rejects a declared value that disagrees, so a receipt can never claim `Proved` while unsigned or after a failed check (absence of a signature returns Speculative, not Proved). Consumes the explicit `attestation.signed` from SP-GATE-002 (source-os #312) and the SyncCycleReceipt attestation from SP-GATE-001. Examples: verified+signed→Proved, verified-but-unsigned→Speculative, failed→Refuted, dry-run→Speculative. Teeth-verified. - Data-Acquisition **citable commons** (WO-COMMONS-2): a promoted platform `CatalogEntry` may carry `promotion.doi` and the approving `DataAcquisitionRequest` records the same `mintedDoi` — a citable DOI (Zenodo/DataCite-style) minted when the governance ∧ IP/legal meet approves promotion. Invariants (schema `allOf` + validator): a DOI may appear only on a `promoted` entry, `mintedDoi` only on an `approved` DAR, and where both are present they must be equal (minted once). Grounds the governed platform tier as a citable open commons. diff --git a/examples/data_class.currency.json b/examples/data_class.currency.json index 5da6589..87ae442 100644 --- a/examples/data_class.currency.json +++ b/examples/data_class.currency.json @@ -32,6 +32,9 @@ "labels": [ "urn:srcos:glossary:revenue", "urn:srcos:glossary:cost" - ] + ], + "head": "logistic", + "evalRunRef": "urn:srcos:run:dataclass-currency-logistic-eval-001", + "threshold": 0.5 } } diff --git a/examples/table_classifier.finance.json b/examples/table_classifier.finance.json new file mode 100644 index 0000000..3cad0df --- /dev/null +++ b/examples/table_classifier.finance.json @@ -0,0 +1,27 @@ +{ + "id": "urn:srcos:table-classifier:finance-ledger-v1", + "type": "TableClassifier", + "specVersion": "2.0.0", + "schemaRef": "urn:srcos:schema-definition:finance-ledger", + "head": "softmax", + "embeddings": [ + { + "kind": "lsa-bag-of-words", + "dimension": 300 + }, + { + "kind": "doc2vec-sentence-encoder", + "dimension": 300 + } + ], + "modelRef": "urn:srcos:model-manifest:table-assigner-softmax-v1", + "version": "1.0.0", + "runRef": "urn:srcos:run:table-assigner-train-001", + "compute": { + "platform": "tritfabric", + "assetRef": "urn:srcos:physical-asset:trit-cluster-01" + }, + "assignsClasses": [ + "urn:srcos:data-class:currency-amount-usd" + ] +} diff --git a/fixtures/data-class/conformance.json b/fixtures/data-class/conformance.json index bae97de..52d31f0 100644 --- a/fixtures/data-class/conformance.json +++ b/fixtures/data-class/conformance.json @@ -39,7 +39,10 @@ "labels": [ "urn:srcos:glossary:revenue", "urn:srcos:glossary:cost" - ] + ], + "head": "logistic", + "evalRunRef": "urn:srcos:run:dataclass-currency-logistic-eval-001", + "threshold": 0.5 } } }, @@ -79,7 +82,10 @@ "labels": [ "urn:srcos:glossary:revenue", "urn:srcos:glossary:cost" - ] + ], + "head": "logistic", + "evalRunRef": "urn:srcos:run:dataclass-currency-logistic-eval-001", + "threshold": 0.5 } } }, @@ -120,7 +126,10 @@ ], "labels": [ "not-a-glossary-urn" - ] + ], + "head": "logistic", + "evalRunRef": "urn:srcos:run:dataclass-currency-logistic-eval-001", + "threshold": 0.5 } } }, @@ -161,9 +170,74 @@ "labels": [ "urn:srcos:glossary:revenue", "urn:srcos:glossary:cost" - ] + ], + "head": "logistic", + "evalRunRef": "urn:srcos:run:dataclass-currency-logistic-eval-001", + "threshold": 0.5 } } + }, + { + "schema": "TableClassifier.json", + "reason": "table classifier head is pinned to softmax (n-ary).", + "failValidator": "const", + "document": { + "id": "urn:srcos:table-classifier:finance-ledger-v1", + "type": "TableClassifier", + "specVersion": "2.0.0", + "schemaRef": "urn:srcos:schema-definition:finance-ledger", + "head": "logistic", + "embeddings": [ + { + "kind": "lsa-bag-of-words", + "dimension": 300 + }, + { + "kind": "doc2vec-sentence-encoder", + "dimension": 300 + } + ], + "modelRef": "urn:srcos:model-manifest:table-assigner-softmax-v1", + "version": "1.0.0", + "runRef": "urn:srcos:run:table-assigner-train-001", + "compute": { + "platform": "tritfabric", + "assetRef": "urn:srcos:physical-asset:trit-cluster-01" + }, + "assignsClasses": [ + "urn:srcos:data-class:currency-amount-usd" + ] + } + }, + { + "schema": "TableClassifier.json", + "reason": "embedding kind is closed to lsa-bag-of-words | doc2vec-sentence-encoder.", + "failValidator": "enum", + "document": { + "id": "urn:srcos:table-classifier:finance-ledger-v1", + "type": "TableClassifier", + "specVersion": "2.0.0", + "schemaRef": "urn:srcos:schema-definition:finance-ledger", + "head": "softmax", + "embeddings": [ + { + "kind": "word2vec" + }, + { + "kind": "doc2vec-sentence-encoder" + } + ], + "modelRef": "urn:srcos:model-manifest:table-assigner-softmax-v1", + "version": "1.0.0", + "runRef": "urn:srcos:run:table-assigner-train-001", + "compute": { + "platform": "tritfabric", + "assetRef": "urn:srcos:physical-asset:trit-cluster-01" + }, + "assignsClasses": [ + "urn:srcos:data-class:currency-amount-usd" + ] + } } ] } diff --git a/schemas/DataClass.json b/schemas/DataClass.json index 0df15cc..a715ef1 100644 --- a/schemas/DataClass.json +++ b/schemas/DataClass.json @@ -54,8 +54,10 @@ "description": "Optional TF-Lattice wide-and-deep classifier that ASSIGNS this data class by inference. It is a first-class cataloged model: modelRef → ModelManifest, runRef → RunRecord, compute on Ray/TritFabric, labels are GlossaryTerms.", "required": [ "kind", + "head", "modelRef", "version", + "evalRunRef", "labels" ], "properties": { @@ -126,6 +128,24 @@ "pattern": "^urn:srcos:glossary:" }, "description": "Output class labels — each a GlossaryTerm URN (labels are assigned in the glossary)." + }, + "head": { + "const": "logistic", + "description": "Per-class head: a LOGISTIC (one-vs-rest, binary) classifier so THIS class / glossary term can be tested INDIVIDUALLY (per-class precision/recall). The table-level n-ary assignment lives in TableClassifier (softmax)." + }, + "evalRunRef": { + "type": "string", + "pattern": "^urn:srcos:run:", + "description": "The per-class evaluation RunRecord — the individual test for this class/label." + }, + "threshold": { + "type": [ + "number", + "null" + ], + "minimum": 0, + "maximum": 1, + "description": "Decision threshold on the logistic probability." } } } diff --git a/schemas/TableClassifier.json b/schemas/TableClassifier.json new file mode 100644 index 0000000..b82eb74 --- /dev/null +++ b/schemas/TableClassifier.json @@ -0,0 +1,121 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://schemas.srcos.ai/v2/TableClassifier.json", + "title": "TableClassifier", + "description": "The per-table (per-SchemaDefinition) SOFTMAX classifier that assigns each column to one of N DataClasses — the n-ary logit→class assignment. It ALIGNS two representations of each column: an LSA bag-of-words embedding (the closed fixed vocabulary) and a doc2vec sentence-encoder (contextual). A first-class cataloged model (ModelManifest) with a run on Ray/TritFabric.", + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "type", + "specVersion", + "head", + "embeddings", + "modelRef", + "assignsClasses" + ], + "properties": { + "id": { + "type": "string", + "pattern": "^urn:srcos:table-classifier:[A-Za-z0-9._~:-]+$" + }, + "type": { + "const": "TableClassifier" + }, + "specVersion": { + "type": "string" + }, + "schemaRef": { + "type": [ + "string", + "null" + ], + "pattern": "^urn:srcos:[A-Za-z0-9._~:-]+$", + "description": "The SchemaDefinition/table this classifier assigns columns for." + }, + "head": { + "const": "softmax", + "description": "n-ary multinomial head — one class per column." + }, + "embeddings": { + "type": "array", + "minItems": 2, + "description": "The representations aligned by the softmax. MUST include both an LSA bag-of-words embedding and a doc2vec sentence-encoder (the n-ary logit→class examples).", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "kind" + ], + "properties": { + "kind": { + "type": "string", + "enum": [ + "lsa-bag-of-words", + "doc2vec-sentence-encoder" + ] + }, + "dimension": { + "type": [ + "integer", + "null" + ], + "minimum": 1 + } + } + } + }, + "modelRef": { + "type": "string", + "pattern": "^urn:srcos:model-manifest:" + }, + "version": { + "type": [ + "string", + "null" + ] + }, + "runRef": { + "type": [ + "string", + "null" + ], + "pattern": "^urn:srcos:run:" + }, + "compute": { + "type": [ + "object", + "null" + ], + "additionalProperties": false, + "required": [ + "platform" + ], + "properties": { + "platform": { + "type": "string", + "enum": [ + "ray", + "tritfabric" + ] + }, + "assetRef": { + "type": [ + "string", + "null" + ], + "pattern": "^urn:srcos:[A-Za-z0-9._~:-]+$" + } + } + }, + "assignsClasses": { + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "pattern": "^urn:srcos:data-class:[A-Za-z0-9._~:-]+$" + }, + "description": "The DataClasses this softmax can assign — the N of the n-ary." + } + } +} diff --git a/specs/data-class-contract.md b/specs/data-class-contract.md index 1faa834..433f745 100644 --- a/specs/data-class-contract.md +++ b/specs/data-class-contract.md @@ -21,3 +21,17 @@ does not fit. ## Conformance `make validate-data-class-examples` (in `make validate`). + +## Two-level classifier architecture + +Assignment is tested at two granularities, so each class is verifiable on its own and the +table is verifiable as a whole: + +- **Per-class — LOGISTIC** (`DataClass.classifier`, `head: logistic`): a one-vs-rest binary + classifier per class / glossary term, with a required `evalRunRef` — the **individual test** + for that class (per-class precision/recall). A class with no eval run is untestable and refused. +- **Per-table — SOFTMAX** (`TableClassifier`, `head: softmax`): the n-ary multinomial that + assigns each column to one of N `DataClass`es. It must **align two representations** of each + column — an **LSA bag-of-words** embedding (the closed fixed vocabulary) and a **doc2vec + sentence-encoder** (contextual) — because these are the n-ary examples of the logit→class + assignment. A cataloged model (`ModelManifest`) with a run on Ray/TritFabric. diff --git a/tools/validate_data_class_examples.py b/tools/validate_data_class_examples.py index 64201d0..5b6ac16 100644 --- a/tools/validate_data_class_examples.py +++ b/tools/validate_data_class_examples.py @@ -76,8 +76,30 @@ def check_classifier(dcs) -> None: FAILURES.append(f"{name}: classifier.modelRef must be a ModelManifest (cataloged model)") elif (clf.get("compute") or {}).get("platform") not in ("ray", "tritfabric"): FAILURES.append(f"{name}: classifier compute must run on ray|tritfabric") + elif clf.get("head") != "logistic": + FAILURES.append(f"{name}: per-class classifier head must be 'logistic' (one-vs-rest) so the class is individually testable") + elif not clf.get("evalRunRef"): + FAILURES.append(f"{name}: per-class classifier must carry an evalRunRef — the individual test for this class/label") else: - CHECKS[f"classifier:{name}:cataloged-on-compute-with-glossary-labels"] = True + CHECKS[f"classifier:{name}:logistic-individually-testable"] = True + + +def check_table_classifiers(tcs) -> None: + """A TableClassifier is the n-ary SOFTMAX; it must align BOTH a LSA bag-of-words and a + doc2vec sentence-encoder, and be a cataloged model on ray/tritfabric assigning N DataClasses.""" + for name, tc in tcs.items(): + kinds = {e.get("kind") for e in (tc.get("embeddings") or [])} + if tc.get("head") != "softmax": + FAILURES.append(f"{name}: table classifier head must be 'softmax' (n-ary)") + elif not {"lsa-bag-of-words", "doc2vec-sentence-encoder"} <= kinds: + FAILURES.append(f"{name}: softmax must align BOTH lsa-bag-of-words AND doc2vec-sentence-encoder " + f"(the n-ary logit→class representations); have {sorted(kinds)}") + elif not tc.get("modelRef", "").startswith("urn:srcos:model-manifest:"): + FAILURES.append(f"{name}: modelRef must be a cataloged ModelManifest") + elif not (tc.get("assignsClasses") and all(c.startswith("urn:srcos:data-class:") for c in tc["assignsClasses"])): + FAILURES.append(f"{name}: assignsClasses must be DataClass URNs (the N of the n-ary)") + else: + CHECKS[f"table-classifier:{name}:softmax-lsa+doc2vec-cataloged"] = True def check_field_conformance(dcs, fields) -> None: @@ -101,6 +123,8 @@ def check_field_conformance(dcs, fields) -> None: def check_negatives(dataclass_schema) -> None: fx = load(ROOT / "fixtures" / "data-class" / "conformance.json") for i, case in enumerate(fx["cases"]): + if case["schema"] != "DataClass.json": + continue # TableClassifier negatives handled in main() against their own schema expected = case.get("failValidator") try: validator_for(dataclass_schema).validate(case["document"]) @@ -116,13 +140,37 @@ def check_negatives(dataclass_schema) -> None: def main() -> int: dataclass_schema = load(SCHEMAS / "DataClass.json") field_schema = load(SCHEMAS / "EntityField.json") + table_schema = load(SCHEMAS / "TableClassifier.json") dcs = {"data_class.currency.json": load(ROOT / "examples" / "data_class.currency.json")} fields = {"entity_field.revenue.json": load(ROOT / "examples" / "entity_field.revenue.json")} + tcs = {"table_classifier.finance.json": load(ROOT / "examples" / "table_classifier.finance.json")} check_conformance(dataclass_schema, field_schema, dcs, fields) + for name, tc in tcs.items(): + errs = sorted(validator_for(table_schema).iter_errors(tc), key=str) + if errs: + for e in errs: + FAILURES.append(f"{name}: {e.message}") + else: + CHECKS[f"schema:{name}"] = True check_classifier(dcs) + check_table_classifiers(tcs) check_field_conformance(dcs, fields) check_negatives(dataclass_schema) + # TableClassifier negatives use their own schema + fx = load(ROOT / "fixtures" / "data-class" / "conformance.json") + for i, case in enumerate(fx["cases"]): + if case["schema"] != "TableClassifier.json": + continue + try: + validator_for(table_schema).validate(case["document"]) + FAILURES.append(f"negative {i} (TableClassifier) unexpectedly PASSED: {case['reason']}") + except jsonschema.ValidationError as exc: + exp = case.get("failValidator") + if exp and exc.validator != exp: + FAILURES.append(f"negative {i} (TableClassifier): failed on {exc.validator!r}, not {exp!r}") + else: + CHECKS[f"negative-tc:{i}:{exc.validator}"] = True for m in FAILURES: print(f"FAIL: {m}", file=sys.stderr) From ae208946613fcbb64223ca8f24f798ee5b9a0e58 Mon Sep 17 00:00:00 2001 From: Michael Heller <21163552+mdheller@users.noreply.github.com> Date: Sun, 2 Aug 2026 16:52:05 -0400 Subject: [PATCH 3/4] fix(data): address Copilot review on DataClass (#251) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - DataClass.classifier now REQUIRES runRef + compute and both are non-null — a classifier is a fully cataloged model with a run on ray/tritfabric, not a partial block (fail-closed intent). - EntityField: allOf if/then — dataClassRef present ⇒ validValues required, so schema-only validators also catch a field bound to a class without a declared domain. - validate_data_class_examples.py: validate each collection against its INTENDED schema explicitly (never pick schema from the instance's own type — a mistyped doc could validate against the wrong schema). - spec: reword 'binds four things, fail-closed if any missing' → three required + an optional fourth (assignment), matching the schema/validator. make validate ok; dup-$id 347. --- schemas/DataClass.json | 12 ++++-------- schemas/EntityField.json | 22 +++++++++++++++++++++- specs/data-class-contract.md | 2 +- tools/validate_data_class_examples.py | 18 ++++++++++-------- 4 files changed, 36 insertions(+), 18 deletions(-) diff --git a/schemas/DataClass.json b/schemas/DataClass.json index a715ef1..1fed150 100644 --- a/schemas/DataClass.json +++ b/schemas/DataClass.json @@ -58,6 +58,8 @@ "modelRef", "version", "evalRunRef", + "runRef", + "compute", "labels" ], "properties": { @@ -73,18 +75,12 @@ "type": "string" }, "runRef": { - "type": [ - "string", - "null" - ], + "type": "string", "pattern": "^urn:srcos:run:", "description": "The training/eval RunRecord." }, "compute": { - "type": [ - "object", - "null" - ], + "type": "object", "additionalProperties": false, "required": [ "platform" diff --git a/schemas/EntityField.json b/schemas/EntityField.json index 217c347..7c8dfa8 100644 --- a/schemas/EntityField.json +++ b/schemas/EntityField.json @@ -54,5 +54,25 @@ "pattern": "^urn:srcos:data-class:[A-Za-z0-9._~:-]+$", "description": "Optional binding to the OntoDQ DataClass this field is an instance of. When set, the field's validValues must conform to the DataClass domain (validator)." } - } + }, + "allOf": [ + { + "$comment": "A field bound to a DataClass must declare its validValues, so its domain conformance is checkable.", + "if": { + "required": [ + "dataClassRef" + ], + "properties": { + "dataClassRef": { + "type": "string" + } + } + }, + "then": { + "required": [ + "validValues" + ] + } + } + ] } diff --git a/specs/data-class-contract.md b/specs/data-class-contract.md index 433f745..023a30a 100644 --- a/specs/data-class-contract.md +++ b/specs/data-class-contract.md @@ -3,7 +3,7 @@ The business vocabulary (`GlossaryTerm`) says what a thing MEANS; the `DataClass` says what its DATA is, ontologically — so data quality can be inferred and enforced, not asserted. -A `DataClass` binds four things and is fail-closed if any is missing: +A `DataClass` binds three things and is fail-closed if any is missing, plus an optional fourth (assignment): - **OntoDT** — `ontologyClassRef`: the datatype-ontology class (capture; enables reusable, inferable data-quality logic — IBM OntoDT). - **biz↔data** — `glossaryTermRef`: the business `GlossaryTerm` this data class realizes. diff --git a/tools/validate_data_class_examples.py b/tools/validate_data_class_examples.py index 5b6ac16..976f512 100644 --- a/tools/validate_data_class_examples.py +++ b/tools/validate_data_class_examples.py @@ -53,14 +53,16 @@ def validator_for(schema: dict) -> jsonschema.Draft202012Validator: def check_conformance(dataclass_schema, field_schema, dcs, fields) -> None: - for name, d in {**dcs, **fields}.items(): - schema = dataclass_schema if d.get("type") == "DataClass" else field_schema - errs = sorted(validator_for(schema).iter_errors(d), key=str) - if errs: - for e in errs: - FAILURES.append(f"{name}: {e.message}") - else: - CHECKS[f"schema:{name}"] = True + # Validate each collection against its INTENDED schema explicitly — never pick the + # schema from the instance's own `type`, or a mistyped doc validates against the wrong one. + for collection, schema in ((dcs, dataclass_schema), (fields, field_schema)): + for name, d in collection.items(): + errs = sorted(validator_for(schema).iter_errors(d), key=str) + if errs: + for e in errs: + FAILURES.append(f"{name}: {e.message}") + else: + CHECKS[f"schema:{name}"] = True def check_classifier(dcs) -> None: From 4f1757221e262bbc0683f31e276f8f594ce438ec Mon Sep 17 00:00:00 2001 From: Michael Heller <21163552+mdheller@users.noreply.github.com> Date: Sun, 2 Aug 2026 17:16:02 -0400 Subject: [PATCH 4/4] =?UTF-8?q?fix(data):=20harden=20classifier=20validato?= =?UTF-8?q?r=20=E2=80=94=20independent=20runRef=20check=20+=20type-guards?= =?UTF-8?q?=20(Copilot=20#251)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit check_classifier now verifies classifier.runRef is a RunRecord URN independently (not just via the schema pattern), consistent with how it checks modelRef, and guards against a non-dict classifier/compute so a prior schema failure can't crash the validator. check_table_classifiers skips non-dict embedding entries. The other four review items were already resolved in earlier remediation (validate-by-intended-schema, schema requires runRef+compute, EntityField if/then dataClassRef=>validValues, spec reworded 'three + optional fourth'). Teeth-verified. --- tools/validate_data_class_examples.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/tools/validate_data_class_examples.py b/tools/validate_data_class_examples.py index 976f512..5e77ba0 100644 --- a/tools/validate_data_class_examples.py +++ b/tools/validate_data_class_examples.py @@ -71,17 +71,23 @@ def check_classifier(dcs) -> None: if not clf: CHECKS[f"classifier:{name}:none"] = True continue + if not isinstance(clf, dict): + FAILURES.append(f"{name}: classifier must be an object") # type-guard: don't crash on a bad shape + continue labels = clf.get("labels") or [] + compute = clf.get("compute") if isinstance(clf.get("compute"), dict) else {} if not all(l.startswith("urn:srcos:glossary:") for l in labels): FAILURES.append(f"{name}: classifier labels must all be GlossaryTerm URNs (assigned in the glossary)") - elif not clf.get("modelRef", "").startswith("urn:srcos:model-manifest:"): + elif not str(clf.get("modelRef", "")).startswith("urn:srcos:model-manifest:"): FAILURES.append(f"{name}: classifier.modelRef must be a ModelManifest (cataloged model)") - elif (clf.get("compute") or {}).get("platform") not in ("ray", "tritfabric"): + elif not str(clf.get("runRef", "")).startswith("urn:srcos:run:"): + FAILURES.append(f"{name}: classifier.runRef must be a RunRecord URN (the training run that produced this model)") + elif compute.get("platform") not in ("ray", "tritfabric"): FAILURES.append(f"{name}: classifier compute must run on ray|tritfabric") elif clf.get("head") != "logistic": FAILURES.append(f"{name}: per-class classifier head must be 'logistic' (one-vs-rest) so the class is individually testable") - elif not clf.get("evalRunRef"): - FAILURES.append(f"{name}: per-class classifier must carry an evalRunRef — the individual test for this class/label") + elif not str(clf.get("evalRunRef", "")).startswith("urn:srcos:run:"): + FAILURES.append(f"{name}: per-class classifier must carry an evalRunRef (RunRecord URN) — the individual test for this class/label") else: CHECKS[f"classifier:{name}:logistic-individually-testable"] = True @@ -90,7 +96,7 @@ def check_table_classifiers(tcs) -> None: """A TableClassifier is the n-ary SOFTMAX; it must align BOTH a LSA bag-of-words and a doc2vec sentence-encoder, and be a cataloged model on ray/tritfabric assigning N DataClasses.""" for name, tc in tcs.items(): - kinds = {e.get("kind") for e in (tc.get("embeddings") or [])} + kinds = {e.get("kind") for e in (tc.get("embeddings") or []) if isinstance(e, dict)} if tc.get("head") != "softmax": FAILURES.append(f"{name}: table classifier head must be 'softmax' (n-ary)") elif not {"lsa-bag-of-words", "doc2vec-sentence-encoder"} <= kinds: