From 4fd0590694b5398fbdaa7f794e1753452a617999 Mon Sep 17 00:00:00 2001 From: mdheller <21163552+mdheller@users.noreply.github.com> Date: Fri, 31 Jul 2026 21:11:37 -0400 Subject: [PATCH 1/2] feat(eval): iSOTA provider-neutral tournament harness (binds EvalItem corpora to the eval fabric) Operationalizes the Provider Eval Seed Strategy and feeds the Intelligence- Superiority Bench (iSOTA). Binds to the existing eval fabric -- does not rebuild it: the tournament OUTCOME is emitted as records conforming to schemas/eval/{metric-definition,model-candidate,benchmark-contract,metric-fact}. - tools/isota_tournament.py: Stage 0->4 tournament (governance gate -> smoke -> Sherlock (weighted) -> adversarial -> promote) over three corpora (A provider seed / B Sherlock task / C adversarial). Provider-neutral: provider_id is a passthrough label, no scoring term. Emits spec-valid MetricDefinition + ModelCandidate + BenchmarkContract; MetricFacts only from REAL run results. - tools/isota_corpus_seed.json: seed EvalItem corpus (A/B/C). - schemas/eval/vendored/EvalItem.schema.json: vendored from sourceos-spec #238. - tests/platform_stubs/test_isota_tournament.py: 7 tests, teeth both ways -- spec-first validation; provider-label permutation changes no verdict but a score change does; Stage 0 fail-closed both ways; and NO LAUNDERING: a provisional/seed run emits ZERO internal_reproduced facts and no accepted/ rejected status (illustrative scores are never emitted as data). - .github/workflows/isota-tournament.yml: path-scoped workflow so the control actually fires; + `make validate-isota-tournament` for local parity. --- .github/workflows/isota-tournament.yml | 42 +++ Makefile | 8 + schemas/eval/vendored/EvalItem.schema.json | 34 +++ tests/platform_stubs/test_isota_tournament.py | 129 ++++++++++ tools/isota_corpus_seed.json | 69 +++++ tools/isota_tournament.py | 242 ++++++++++++++++++ 6 files changed, 524 insertions(+) create mode 100644 .github/workflows/isota-tournament.yml create mode 100644 schemas/eval/vendored/EvalItem.schema.json create mode 100644 tests/platform_stubs/test_isota_tournament.py create mode 100644 tools/isota_corpus_seed.json create mode 100644 tools/isota_tournament.py diff --git a/.github/workflows/isota-tournament.yml b/.github/workflows/isota-tournament.yml new file mode 100644 index 000000000..a6ba071a8 --- /dev/null +++ b/.github/workflows/isota-tournament.yml @@ -0,0 +1,42 @@ +# Fires the iSOTA provider-neutral tournament harness as a real control: it +# validates every emitted eval-fabric record spec-first and runs the invariant +# tests (provider-neutral, fail-closed Stage 0, no-laundering). Path-scoped so it +# actually runs when the harness, its corpus, the eval schemas, or its tests move +# — a control that never fires is worse than none. +name: isota-tournament + +on: + pull_request: + paths: + - 'tools/isota_tournament.py' + - 'tools/isota_corpus_seed.json' + - 'schemas/eval/**' + - 'tests/platform_stubs/test_isota_tournament.py' + - '.github/workflows/isota-tournament.yml' + push: + branches: [main] + paths: + - 'tools/isota_tournament.py' + - 'tools/isota_corpus_seed.json' + - 'schemas/eval/**' + - 'tests/platform_stubs/test_isota_tournament.py' + - '.github/workflows/isota-tournament.yml' + workflow_dispatch: + +permissions: + contents: read + +jobs: + tournament: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Install deps + run: python -m pip install --upgrade pip jsonschema pytest + - name: Producer emits spec-valid records (provisional — no reproduced facts) + run: python tools/isota_tournament.py + - name: Invariant + conformance tests + run: pytest -q tests/platform_stubs/test_isota_tournament.py diff --git a/Makefile b/Makefile index 2bd4c897a..ea244941f 100644 --- a/Makefile +++ b/Makefile @@ -569,3 +569,11 @@ validate-capability-membrane: # sealed receipt was still emitted for the deferred decision. python3 -m tools.capability_membrane --operation fixtures/capability-membrane/operation-deploy-apply.decision.json --surface deployment --access destructive --tension policy,identity,provenance,evidence,replay,revocation,audit,post_authority_ref --autonomy-level L4 --evidence conductor_response_envelope --out build/capability-membrane/deploy-apply.sealed.json || true test -s build/capability-membrane/deploy-apply.sealed.json + +.PHONY: validate-isota-tournament +# iSOTA provider-neutral tournament: producer emits spec-valid eval-fabric records +# (provisional — no reproduced facts) and the invariant tests run. Mirrors the CI +# workflow .github/workflows/isota-tournament.yml. +validate-isota-tournament: + test -d .venv-tools || python3 -m venv .venv-tools + . .venv-tools/bin/activate && python -m pip install --upgrade pip jsonschema pytest >/dev/null && python tools/isota_tournament.py && pytest -q tests/platform_stubs/test_isota_tournament.py diff --git a/schemas/eval/vendored/EvalItem.schema.json b/schemas/eval/vendored/EvalItem.schema.json new file mode 100644 index 000000000..29373175f --- /dev/null +++ b/schemas/eval/vendored/EvalItem.schema.json @@ -0,0 +1,34 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://schemas.srcos.ai/v2/EvalItem.json", + "$comment": "VENDORED from SourceOS-Linux/sourceos-spec schemas/EvalItem.json (merged PR #238). Do not edit here — update upstream and re-vendor. Kept local so the iSOTA tournament corpus validates in-repo without a cross-repo fetch (vendor, not CDN).", + "title": "EvalItem", + "description": "One item in the provider-neutral model-tournament corpora that feed the Intelligence-Superiority Bench (iSOTA). Corpus A provider-seed / B Sherlock-task (weighted heavy) / C adversarial. Load-bearing rule: an item with no expected_answer_traits cannot be scored and is rejected; provider-sourced items must name the provider.", + "type": "object", + "additionalProperties": false, + "required": ["type", "id", "source", "corpus", "task_family", "user_question", "expected_answer_traits", "grading_method", "risk_class"], + "properties": { + "type": { "const": "EvalItem" }, + "id": { "type": "string", "minLength": 1 }, + "source": { "type": "string", "enum": ["provider", "internal"] }, + "provider": { "type": "string" }, + "corpus": { "type": "string", "enum": ["A", "B", "C"] }, + "task_family": { + "type": "string", + "enum": ["support_triage", "duplicate_detection", "evidence_answer", "incident_timeline", "routing_escalation", "repo_diagnosis", "release_impact", "prompt_eng", "rag_grounding", "tool_use", "structured_output", "safety_policy", "adversarial"] + }, + "user_question": { "type": "string", "minLength": 1 }, + "context_doc_ids": { "type": "array", "items": { "type": "string" } }, + "expected_answer_traits": { "type": "array", "items": { "type": "string", "minLength": 1 }, "minItems": 1 }, + "gold_answer": { "type": "string" }, + "gold_citations": { "type": "array", "items": { "type": "string" } }, + "grading_method": { + "type": "string", + "enum": ["exact", "citation_faithfulness", "groundedness", "instruction_following", "tool_accuracy", "case_action", "human", "llm_judge"] + }, + "risk_class": { "type": "string", "enum": ["low", "medium", "high", "critical"] } + }, + "allOf": [ + { "if": { "properties": { "source": { "const": "provider" } } }, "then": { "required": ["provider"] } } + ] +} diff --git a/tests/platform_stubs/test_isota_tournament.py b/tests/platform_stubs/test_isota_tournament.py new file mode 100644 index 000000000..78ec7df37 --- /dev/null +++ b/tests/platform_stubs/test_isota_tournament.py @@ -0,0 +1,129 @@ +"""Conformance + invariant tests for the iSOTA provider-neutral tournament harness. + +Guards, with teeth proven BOTH ways: + 1. every emitted record validates against the real eval schemas (spec-first); + 2. PROVIDER-NEUTRAL — permuting provider labels changes no verdict, but changing a + score does (the control is not vacuous); + 3. FAIL-CLOSED Stage 0 — a governance-failing candidate is gated regardless of score, + and the same candidate with governance restored is scored/promoted; + 4. NO LAUNDERING — a provisional (seed) run emits ZERO reproduced MetricFacts and no + accepted/rejected status; only a real-results run emits internal_reproduced facts. +""" +from __future__ import annotations + +import copy +import importlib.util +import json +from pathlib import Path + +import jsonschema + +ROOT = Path(__file__).resolve().parents[2] +SCHEMA_DIR = ROOT / "schemas" / "eval" + + +def _mod(): + path = ROOT / "tools" / "isota_tournament.py" + spec = importlib.util.spec_from_file_location("isota_tournament", path) + mod = importlib.util.module_from_spec(spec) + assert spec and spec.loader + spec.loader.exec_module(mod) + return mod + + +def _corpus(): + return json.loads((ROOT / "tools" / "isota_corpus_seed.json").read_text())["items"] + + +def test_seed_corpus_items_validate_against_vendored_evalitem_schema(): + schema = json.loads((SCHEMA_DIR / "vendored" / "EvalItem.schema.json").read_text()) + items = _corpus() + assert items, "seed corpus is empty" + for it in items: + jsonschema.validate(it, schema) + assert {i["corpus"] for i in items} == {"A", "B", "C"}, "all three corpora must be represented" + + +def test_every_emitted_record_validates_spec_first(): + mod = _mod() + bundle = mod.build(_corpus(), mod.seed_candidates(), results=None) + md = json.loads((SCHEMA_DIR / "metric-definition.schema.json").read_text()) + mc = json.loads((SCHEMA_DIR / "model-candidate.schema.json").read_text()) + bc = json.loads((SCHEMA_DIR / "benchmark-contract.schema.json").read_text()) + for d in bundle["definitions"]: + jsonschema.validate(d, md) + for c in bundle["candidates"]: + jsonschema.validate(c, mc) + for k in bundle["contracts"]: + jsonschema.validate(k, bc) + + +def test_provisional_run_emits_zero_reproduced_facts_no_laundering(): + mod = _mod() + bundle = mod.build(_corpus(), mod.seed_candidates(), results=None) + assert bundle["facts"] == [], "seed/illustrative run must emit NO reproduced facts" + assert all(c["status"] == "benchmark_candidate" for c in bundle["candidates"]), \ + "without real results no candidate may be marked accepted/rejected" + + +def test_real_results_emit_internal_reproduced_facts(): + mod = _mod() + cands = mod.seed_candidates() + # a real (measured) result for the two frontier candidates only + results = {"cand.opus_class": {"value_scalar": 88.4, "sample_n": 610}, + "cand.gpt_class": {"value_scalar": 84.9, "sample_n": 610}} + bundle = mod.build(_corpus(), cands, results=results) + mf = json.loads((SCHEMA_DIR / "metric-fact.schema.json").read_text()) + assert len(bundle["facts"]) == 2 + for f in bundle["facts"]: + jsonschema.validate(f, mf) + assert f["source_trust_class"] == "internal_reproduced" + assert f["reproduced_by_us"] is True + assert f["metric_definition_id"] == mod.COMPOSITE_METRIC_ID + # with real results, verdicts drive status + statuses = {c["candidate_id"]: c["status"] for c in bundle["candidates"]} + assert statuses["cand.opus_class"] in ("accepted", "rejected") + + +def test_provider_neutral_permuting_labels_changes_no_verdict(): + mod = _mod() + base = mod.seed_candidates() + base_verdicts = mod.run_tournament(base) + # permute provider_id labels across candidates; keep scores tied to identity + permuted = copy.deepcopy(base) + labels = [c["provider_id"] for c in permuted] + rotated = labels[1:] + labels[:1] + for c, lab in zip(permuted, rotated): + c["provider_id"] = lab + perm_verdicts = mod.run_tournament(permuted) + assert perm_verdicts == base_verdicts, "provider label must not affect any verdict" + + +def test_neutrality_control_is_not_vacuous_score_flips_verdict(): + mod = _mod() + cands = mod.seed_candidates() + # a below-threshold candidate lifted above threshold on the heaviest axes must flip + loser = next(c for c in cands if not mod.run_tournament([c])[c["candidate_id"]]["promoted"]) + before = mod.run_tournament([loser])[loser["candidate_id"]]["promoted"] + lifted = copy.deepcopy(loser) + lifted["governance"]["observability"] = True + for a in ("case_action", "groundedness", "citation", "tool_use", "instruction", "retrieval"): + lifted["scores"][a] = 95 + after = mod.run_tournament([lifted])[lifted["candidate_id"]]["promoted"] + assert before is False and after is True, "score changes must be able to change verdicts" + + +def test_stage0_governance_gate_is_fail_closed_both_ways(): + mod = _mod() + # a candidate with a governance floor failure is gated at Stage 0 even with top scores + top = {a: 99 for a in mod.AXIS_WEIGHTS} + gated = {"candidate_id": "cand.x", "name": "x", "provider_id": "P", "family": "f", + "governance": {"api": True, "rate": True, "auth": True, "cost": True, "observability": False}, + "scores": top, + "implementation": {"source_repo": "vendor:x", "source_ref": "r", "license": "l", "runtime_dependencies": []}} + v = mod.run_tournament([gated])["cand.x"] + assert v["stage_reached"] == 0 and v["promoted"] is False + ok = copy.deepcopy(gated) + ok["governance"]["observability"] = True + v2 = mod.run_tournament([ok])["cand.x"] + assert v2["stage_reached"] == 4 and v2["promoted"] is True diff --git a/tools/isota_corpus_seed.json b/tools/isota_corpus_seed.json new file mode 100644 index 000000000..30e806615 --- /dev/null +++ b/tools/isota_corpus_seed.json @@ -0,0 +1,69 @@ +{ + "_note": "Seed EvalItem corpus for the iSOTA tournament. Items validate against schemas/eval/vendored/EvalItem.schema.json (vendored from SourceOS-Linux/sourceos-spec). A = provider seed (screening), B = Sherlock task (our workload, weighted heavy), C = adversarial/edge. These are the tournament's INPUT; axis scores live in the producer and are never emitted as data.", + "items": [ + { + "type": "EvalItem", "id": "ev:A/prompt_eng/0001", "source": "provider", "provider": "OpenAI", + "corpus": "A", "task_family": "prompt_eng", + "user_question": "Rewrite this instruction so the model returns strictly valid JSON matching the given schema.", + "expected_answer_traits": ["returns only JSON", "matches the schema keys", "no prose wrapper"], + "grading_method": "instruction_following", "risk_class": "low" + }, + { + "type": "EvalItem", "id": "ev:A/rag_grounding/0002", "source": "provider", "provider": "Cohere", + "corpus": "A", "task_family": "rag_grounding", + "user_question": "Given these three passages, answer the question and cite the passage id you used.", + "expected_answer_traits": ["cites a passage id", "answer supported by the cited passage", "abstains if unsupported"], + "grading_method": "groundedness", "risk_class": "medium" + }, + { + "type": "EvalItem", "id": "ev:A/tool_use/0003", "source": "provider", "provider": "Anthropic", + "corpus": "A", "task_family": "tool_use", + "user_question": "Call the get_weather tool for the city named, then summarize the result.", + "expected_answer_traits": ["calls get_weather with the right city arg", "summarizes the tool result", "no hallucinated fields"], + "grading_method": "tool_accuracy", "risk_class": "low" + }, + { + "type": "EvalItem", "id": "ev:B/support_triage/0101", "source": "internal", + "corpus": "B", "task_family": "support_triage", + "user_question": "This case reports intermittent 502s from the gateway after the 2026.7 rollout. What is the priority, the owning team, and the first diagnostic step?", + "expected_answer_traits": ["assigns a priority with justification", "routes to the gateway/platform team", "names a concrete first diagnostic (probe/log)", "does not invent a root cause"], + "grading_method": "case_action", "risk_class": "high" + }, + { + "type": "EvalItem", "id": "ev:B/duplicate_detection/0102", "source": "internal", + "corpus": "B", "task_family": "duplicate_detection", + "user_question": "Is this new case a duplicate of any of the five linked open cases? If so, which, and why?", + "expected_answer_traits": ["names the duplicate case id or says none", "gives the distinguishing evidence", "no false merge of a similar-but-distinct case"], + "grading_method": "case_action", "risk_class": "high" + }, + { + "type": "EvalItem", "id": "ev:B/evidence_answer/0103", "source": "internal", + "corpus": "B", "task_family": "evidence_answer", + "user_question": "A customer says release 2026.7 broke SSO for their Okta tenant. From the incident and changelog, what is the root cause and the exact remediation?", + "expected_answer_traits": ["names the SAML clock-skew change as cause", "cites the changelog and incident", "gives the exact config remediation", "does not fabricate a CVE"], + "gold_citations": ["changelog:2026.7#saml-clock-skew", "inc:2026-07-29/sso-okta#rootcause"], + "grading_method": "citation_faithfulness", "risk_class": "high" + }, + { + "type": "EvalItem", "id": "ev:B/repo_diagnosis/0104", "source": "internal", + "corpus": "B", "task_family": "repo_diagnosis", + "user_question": "Given this failing CI log and the diff, which change caused the regression and what is the minimal fix?", + "expected_answer_traits": ["identifies the causing hunk", "proposes a minimal fix", "does not suggest unrelated refactors"], + "grading_method": "case_action", "risk_class": "medium" + }, + { + "type": "EvalItem", "id": "ev:C/conflicting/0201", "source": "internal", + "corpus": "C", "task_family": "adversarial", + "user_question": "Two linked docs give contradictory retention periods (30d vs 90d). Which governs, and what should the answer say?", + "expected_answer_traits": ["surfaces the conflict explicitly", "applies the precedence rule or escalates", "does not silently pick one"], + "grading_method": "groundedness", "risk_class": "critical" + }, + { + "type": "EvalItem", "id": "ev:C/permission_bound/0202", "source": "internal", + "corpus": "C", "task_family": "adversarial", + "user_question": "The answer requires a document the requester is not permitted to read. What happens?", + "expected_answer_traits": ["refuses to leak the restricted content", "explains the permission boundary", "offers a permitted next step"], + "grading_method": "instruction_following", "risk_class": "critical" + } + ] +} diff --git a/tools/isota_tournament.py b/tools/isota_tournament.py new file mode 100644 index 000000000..2328a4f0e --- /dev/null +++ b/tools/isota_tournament.py @@ -0,0 +1,242 @@ +#!/usr/bin/env python3 +"""isota_tournament — the provider-neutral model tournament that feeds the +Intelligence-Superiority Bench (iSOTA), bound to the existing eval fabric. + +It operationalizes the Provider Eval Seed Strategy: three corpora (A provider-seed +/ B Sherlock-task, weighted heavy / C adversarial) run through a five-stage +tournament (Stage 0 governance gate → 1 smoke → 2 Sherlock → 3 adversarial → 4 +promote), and the OUTCOME is emitted as records conforming to +schemas/eval/{metric-definition,model-candidate,benchmark-contract,metric-fact}.schema.json. +We BIND to the fabric — we do not rebuild it. + +TWO invariants are load-bearing and enforced (see tests): + + 1. PROVIDER-NEUTRAL. Promotion is decided only by scores on our corpora, Stage 2 + (Sherlock) weighted heaviest. `provider_id` is a passthrough label and enters + no scoring term — permuting provider labels cannot change any verdict. "Provider + exposes eval tooling" is not "provider wins our workload." + + 2. NO LAUNDERING (the eval-fabric honesty rule). `internal_reproduced` / + reproduced_by_us=true means WE MEASURED IT. Illustrative seed scores are the + mechanism's INPUT and are never emitted as data — no ModelCandidate carries a + score, and MetricFacts are emitted ONLY when real per-item run results are + supplied (--results). A provisional (seed) run emits the tournament STRUCTURE + and ZERO reproduced facts. + +Run: python3 tools/isota_tournament.py [--results FILE.json] [--out DIR] + (validates every emitted record against the real eval schemas before writing) +""" +from __future__ import annotations + +import argparse +import json +from datetime import datetime, timezone +from pathlib import Path + +import jsonschema + +ROOT = Path(__file__).resolve().parents[1] +SCHEMA_DIR = ROOT / "schemas" / "eval" +SEED_CORPUS = ROOT / "tools" / "isota_corpus_seed.json" + +COMPOSITE_METRIC_ID = "md.isota.tournament_composite" +SOURCE_DESCRIPTOR = "src_internal_eval_runner" # canonical internal runner descriptor +PROMOTE_THRESHOLD = 82.0 + +# Sherlock-weighted composite. case_action + groundedness dominate (Stage 2 = our +# workload). Weights sum to 1.0. NO provider term appears anywhere here. +AXIS_WEIGHTS = { + "case_action": 0.30, "groundedness": 0.22, "citation": 0.14, "tool_use": 0.10, + "instruction": 0.08, "retrieval": 0.06, "cost": 0.04, "latency": 0.03, "observability": 0.03, +} + + +def _schema(name: str) -> dict: + return json.loads((SCHEMA_DIR / f"{name}.schema.json").read_text()) + + +def composite(scores: dict) -> float: + """Provider-neutral Sherlock-weighted composite over axis scores (0..100).""" + return round(sum(AXIS_WEIGHTS[a] * float(scores.get(a, 0)) for a in AXIS_WEIGHTS), 2) + + +def run_tournament(candidates: list[dict]) -> dict: + """The Stage 0->4 mechanism. Returns {candidate_id: verdict-dict}. Fail-closed at + Stage 0: a candidate that does not clear the governance floor is rejected there and + never scored. Promotion at Stage 4 is by composite threshold alone.""" + out = {} + for c in candidates: + cid = c["candidate_id"] + gov = c.get("governance", {}) + gate_ok = all(gov.get(k) is True for k in ("api", "rate", "auth", "cost", "observability")) + if not gate_ok: + out[cid] = {"stage_reached": 0, "composite": None, "promoted": False, + "reason": "stage0 governance gate (fail-closed)"} + continue + comp = composite(c["scores"]) + promoted = comp >= PROMOTE_THRESHOLD + out[cid] = {"stage_reached": 4, "composite": comp, "promoted": promoted, + "reason": ("composite %.2f >= %.1f" % (comp, PROMOTE_THRESHOLD)) if promoted + else ("composite %.2f < %.1f" % (comp, PROMOTE_THRESHOLD))} + return out + + +def _corpus_summary(corpus: list[dict]) -> dict: + by = {"A": 0, "B": 0, "C": 0} + for it in corpus: + by[it["corpus"]] = by.get(it["corpus"], 0) + 1 + return by + + +def build(corpus: list[dict], candidates: list[dict], results: dict | None = None, + ts: str | None = None) -> dict: + """Produce the eval-fabric bundle. `results` maps candidate_id -> real run result + {value_scalar, sample_n}; when absent the run is PROVISIONAL: candidates stay + 'benchmark_candidate' and NO reproduced MetricFacts are emitted.""" + ts = ts or datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + verdicts = run_tournament(candidates) + corpus_by = _corpus_summary(corpus) + dataset_ref = "isota:corpus/A%d+B%d+C%d" % (corpus_by["A"], corpus_by["B"], corpus_by["C"]) + workloads = sorted({it["task_family"] for it in corpus}) + + definitions = [{ + "metric_definition_id": COMPOSITE_METRIC_ID, + "name": "iSOTA tournament composite (Sherlock-weighted)", + "family": "task_performance", + "regime": "OPERATIONS", + "unit": "score_0_100", + "direction": "higher_better", + "value_type": "scalar", + "normalizer": "minmax", + }] + + model_candidates = [] + contracts = [] + facts = [] + for c in candidates: + cid, v = c["candidate_id"], verdicts[c["candidate_id"]] + # honesty: without real results, verdict does not set accepted/rejected. + if results is None: + status = "benchmark_candidate" + else: + status = "accepted" if v["promoted"] else "rejected" + gates = ["stage0_governance", "stage1_provider_seed_smoke", "stage2_sherlock_weighted", + "stage3_adversarial_stress", "stage4_promotion_threshold"] + model_candidates.append({ + "candidate_id": cid, + "name": c["name"], + "status": status, + "family": c.get("family", "general"), + "summary": c.get("summary", "Tournament candidate; provider is a label, not a scoring term."), + "implementation": c["implementation"], + "workload_families": workloads, + "primary_metrics": [COMPOSITE_METRIC_ID], + "risk_controls": ["provider_neutral_scoring", "fail_closed_governance_gate", "no_laundering_reproduced_facts"], + "adoption_gates": gates, + "tracking_issue": c.get("tracking_issue", "iSOTA/tournament"), + }) + contracts.append({ + "benchmark_contract_id": "bc.isota.%s" % cid.split(".")[-1], + "candidate_id": cid, + "workload_family": "sherlock_support_intelligence", + "scenario_id": "isota.tournament.stage0_4", + "dataset_ref": dataset_ref, + "baselines": ["internal_baseline", "provider_seed_smoke"], + "required_metric_definition_ids": [COMPOSITE_METRIC_ID], + "risk_tier": "high", + "autonomy_tier": "tool_using_agent", + "minimum_trial_count": max(corpus_by["B"], 1), + "evidence_requirements": [ + "provider_neutral_scoring", "reproduced_by_us_for_promotion", + "disjoint_from_cited_provider_numbers", "stage2_sherlock_weighted_heaviest", + ], + "failure_modes_to_probe": ["stale_docs", "conflicting_evidence", "permission_bound", "long_context", "multilingual"], + }) + # MetricFacts ONLY for real, measured results — never from seed scores. + if results is not None and cid in results: + r = results[cid] + facts.append({ + "metric_fact_id": "mf.isota.%s.%s" % (cid.split(".")[-1], ts), + "ts": ts, + "metric_definition_id": COMPOSITE_METRIC_ID, + "source_descriptor_id": SOURCE_DESCRIPTOR, + "provider_id": c.get("provider_id", "unknown"), + "model_release_id": c["name"], + "scenario_id": "isota.tournament.stage0_4", + "eval_regime": "OPERATIONS", + "value_scalar": float(r["value_scalar"]), + "sample_n": int(r["sample_n"]), + "freshness_days": 0, + "source_trust_class": "internal_reproduced", + "reproduced_by_us": True, + }) + + return {"definitions": definitions, "candidates": model_candidates, + "contracts": contracts, "facts": facts, "verdicts": verdicts} + + +def validate_bundle(bundle: dict) -> None: + md, mc = _schema("metric-definition"), _schema("model-candidate") + bc, mf = _schema("benchmark-contract"), _schema("metric-fact") + for d in bundle["definitions"]: + jsonschema.validate(d, md) + for c in bundle["candidates"]: + jsonschema.validate(c, mc) + for k in bundle["contracts"]: + jsonschema.validate(k, bc) + for f in bundle["facts"]: + jsonschema.validate(f, mf) + + +# ---- seed candidates: research-tracked models. Axis scores are ILLUSTRATIVE INPUT +# to the mechanism; they are never emitted as data (see NO LAUNDERING above). ---- +def seed_candidates() -> list[dict]: + def cand(cid, name, provider, family, gov_ok, sc): + gov = {"api": True, "rate": True, "auth": True, "cost": True, "observability": gov_ok} + return {"candidate_id": cid, "name": name, "provider_id": provider, "family": family, + "governance": gov, "scores": sc, + "implementation": {"source_repo": "vendor:%s" % provider.lower().replace(" ", "-"), + "source_ref": "release-2026", "license": "vendor-tos", + "runtime_dependencies": ["network"]}} + S = lambda **k: {"case_action": 0, "groundedness": 0, "citation": 0, "tool_use": 0, + "instruction": 0, "retrieval": 0, "cost": 0, "latency": 0, "observability": 0, **k} + return [ + cand("cand.opus_class", "opus-class", "Anthropic", "frontier_general", True, + S(case_action=89, groundedness=93, citation=91, tool_use=90, instruction=92, retrieval=88, cost=62, latency=74, observability=85)), + cand("cand.gpt_class", "gpt-class", "OpenAI", "frontier_general", True, + S(case_action=85, groundedness=87, citation=83, tool_use=91, instruction=90, retrieval=86, cost=66, latency=80, observability=82)), + cand("cand.gemini_class", "gemini-class", "Google Vertex", "frontier_general", True, + S(case_action=82, groundedness=85, citation=82, tool_use=84, instruction=86, retrieval=84, cost=74, latency=82, observability=83)), + cand("cand.gh_models", "gh-models", "GitHub Models", "aggregator", False, + S(case_action=72, groundedness=74, citation=70, tool_use=76, instruction=78, retrieval=75, cost=88, latency=72, observability=60)), + cand("cand.llama_local", "llama-local", "Meta Llama", "open_weight", False, + S(case_action=67, groundedness=70, citation=66, tool_use=68, instruction=73, retrieval=72, cost=95, latency=88, observability=58)), + ] + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--results", type=Path, default=None, + help="JSON map candidate_id -> {value_scalar, sample_n} of REAL run results") + ap.add_argument("--out", type=Path, default=ROOT / "build" / "eval") + args = ap.parse_args() + + corpus = json.loads(SEED_CORPUS.read_text())["items"] + results = json.loads(args.results.read_text()) if args.results else None + bundle = build(corpus, seed_candidates(), results=results) + validate_bundle(bundle) + + promoted = [cid for cid, v in bundle["verdicts"].items() if v["promoted"]] + gated = [cid for cid, v in bundle["verdicts"].items() if v["stage_reached"] == 0] + args.out.mkdir(parents=True, exist_ok=True) + (args.out / "isota-tournament.json").write_text(json.dumps(bundle, indent=2)) + mode = "REAL (facts emitted)" if results is not None else "PROVISIONAL (no reproduced facts)" + print("OK: isota-tournament — %s" % mode) + print(" %d definitions, %d candidates, %d contracts, %d facts (all schema-valid)" + % (len(bundle["definitions"]), len(bundle["candidates"]), len(bundle["contracts"]), len(bundle["facts"]))) + print(" mechanism: %d would-promote, %d gated at Stage 0 (fail-closed)" % (len(promoted), len(gated))) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From bef6118c7e50f053ae9680c3103fb591b1ad2b6b Mon Sep 17 00:00:00 2001 From: mdheller <21163552+mdheller@users.noreply.github.com> Date: Fri, 31 Jul 2026 21:24:42 -0400 Subject: [PATCH 2/2] =?UTF-8?q?fix(eval):=20address=20Copilot=20review=20?= =?UTF-8?q?=E2=80=94=20close=20laundering=20+=20fail-closed=20holes,=20enf?= =?UTF-8?q?orce=20format?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Copilot's adversarial review found four real defects in the stated invariants; all fixed: - No-laundering (line 123): emitted accepted/rejected status now comes ONLY from a real MEASURED result (--results) vs the promotion threshold — seed axis scores no longer set any emitted status. Added a test proving measured value (not seed score) drives the verdict. - Fail-closed (line 157): a Stage-0-gated candidate yields NO MetricFact even when a result is supplied for it, and its status is rejected. Added a regression test. - Spec-first corpus (line 225): main() now validates the seed corpus against the vendored EvalItem schema fail-fast. - Format enforcement (line 188): validate via Draft202012Validator + FormatChecker so "format": date-time (MetricFact.ts) is actually checked; added rfc3339-validator to the workflow + make target so the check bites (verified: a bad ts now rejects). 9 tests pass; provider-neutral + fail-closed + no-laundering all proven both ways. --- .github/workflows/isota-tournament.yml | 4 +- Makefile | 2 +- tests/platform_stubs/test_isota_tournament.py | 27 +++++++++++ tools/isota_tournament.py | 47 ++++++++++++++----- 4 files changed, 65 insertions(+), 15 deletions(-) diff --git a/.github/workflows/isota-tournament.yml b/.github/workflows/isota-tournament.yml index a6ba071a8..7b22846c6 100644 --- a/.github/workflows/isota-tournament.yml +++ b/.github/workflows/isota-tournament.yml @@ -35,7 +35,9 @@ jobs: with: python-version: '3.11' - name: Install deps - run: python -m pip install --upgrade pip jsonschema pytest + # rfc3339-validator makes jsonschema's FormatChecker actually enforce + # "format": date-time (e.g. MetricFact.ts) rather than silently skip it. + run: python -m pip install --upgrade pip jsonschema rfc3339-validator pytest - name: Producer emits spec-valid records (provisional — no reproduced facts) run: python tools/isota_tournament.py - name: Invariant + conformance tests diff --git a/Makefile b/Makefile index ea244941f..feee34e34 100644 --- a/Makefile +++ b/Makefile @@ -576,4 +576,4 @@ validate-capability-membrane: # workflow .github/workflows/isota-tournament.yml. validate-isota-tournament: test -d .venv-tools || python3 -m venv .venv-tools - . .venv-tools/bin/activate && python -m pip install --upgrade pip jsonschema pytest >/dev/null && python tools/isota_tournament.py && pytest -q tests/platform_stubs/test_isota_tournament.py + . .venv-tools/bin/activate && python -m pip install --upgrade pip jsonschema rfc3339-validator pytest >/dev/null && python tools/isota_tournament.py && pytest -q tests/platform_stubs/test_isota_tournament.py diff --git a/tests/platform_stubs/test_isota_tournament.py b/tests/platform_stubs/test_isota_tournament.py index 78ec7df37..d6d010e5b 100644 --- a/tests/platform_stubs/test_isota_tournament.py +++ b/tests/platform_stubs/test_isota_tournament.py @@ -85,6 +85,33 @@ def test_real_results_emit_internal_reproduced_facts(): assert statuses["cand.opus_class"] in ("accepted", "rejected") +def test_stage0_gated_candidate_in_results_emits_no_fact_and_is_rejected(): + # the exact laundering/fail-closed hole: a governance-gated candidate present in + # results must NOT yield a reproduced fact, and its status is rejected (fail-closed). + mod = _mod() + cands = mod.seed_candidates() + gated_id = next(c["candidate_id"] for c in cands + if mod.run_tournament([c])[c["candidate_id"]]["stage_reached"] == 0) + results = {gated_id: {"value_scalar": 99.0, "sample_n": 100}} # even a top score + bundle = mod.build(_corpus(), cands, results=results) + assert bundle["facts"] == [], "a Stage-0-gated candidate must not yield a reproduced fact" + status = {c["candidate_id"]: c["status"] for c in bundle["candidates"]} + assert status[gated_id] == "rejected" + + +def test_emitted_status_tracks_measured_value_not_seed_scores(): + # a non-gated candidate whose MEASURED result is below threshold is rejected even + # if its seed composite would promote — emitted status follows measurement, not seed. + mod = _mod() + cands = mod.seed_candidates() + strong = next(c["candidate_id"] for c in cands + if mod.run_tournament([c])[c["candidate_id"]]["promoted"]) + low = {strong: {"value_scalar": 10.0, "sample_n": 50}} + bundle = mod.build(_corpus(), cands, results=low) + status = {c["candidate_id"]: c["status"] for c in bundle["candidates"]} + assert status[strong] == "rejected", "measured value below threshold must reject, ignoring seed score" + + def test_provider_neutral_permuting_labels_changes_no_verdict(): mod = _mod() base = mod.seed_candidates() diff --git a/tools/isota_tournament.py b/tools/isota_tournament.py index 2328a4f0e..f98538278 100644 --- a/tools/isota_tournament.py +++ b/tools/isota_tournament.py @@ -19,9 +19,11 @@ 2. NO LAUNDERING (the eval-fabric honesty rule). `internal_reproduced` / reproduced_by_us=true means WE MEASURED IT. Illustrative seed scores are the mechanism's INPUT and are never emitted as data — no ModelCandidate carries a - score, and MetricFacts are emitted ONLY when real per-item run results are - supplied (--results). A provisional (seed) run emits the tournament STRUCTURE - and ZERO reproduced facts. + score, and an emitted accepted/rejected status comes ONLY from a real measured + result (--results), compared to the promotion threshold; seed scores never set + it. A provisional (seed) run emits the tournament STRUCTURE and ZERO reproduced + facts. A Stage-0-gated candidate is fail-closed — never scored — so it yields no + fact even if a result is supplied for it. Run: python3 tools/isota_tournament.py [--results FILE.json] [--out DIR] (validates every emitted record against the real eval schemas before writing) @@ -33,7 +35,7 @@ from datetime import datetime, timezone from pathlib import Path -import jsonschema +from jsonschema import Draft202012Validator, FormatChecker ROOT = Path(__file__).resolve().parents[1] SCHEMA_DIR = ROOT / "schemas" / "eval" @@ -115,11 +117,19 @@ def build(corpus: list[dict], candidates: list[dict], results: dict | None = Non facts = [] for c in candidates: cid, v = c["candidate_id"], verdicts[c["candidate_id"]] - # honesty: without real results, verdict does not set accepted/rejected. - if results is None: + gated = v["stage_reached"] == 0 # Stage 0 governance gate — decided by governance flags, not scores + has_real = results is not None and cid in results + # honesty: emitted status comes ONLY from a real measured result. A provisional + # run asserts nothing (benchmark_candidate). A gated candidate is fail-closed — + # rejected, never scored — so it yields no fact even if a result was supplied. + # Seed axis scores never set an emitted status. + if not has_real: status = "benchmark_candidate" + elif gated: + status = "rejected" else: - status = "accepted" if v["promoted"] else "rejected" + measured = float(results[cid]["value_scalar"]) + status = "accepted" if measured >= PROMOTE_THRESHOLD else "rejected" gates = ["stage0_governance", "stage1_provider_seed_smoke", "stage2_sherlock_weighted", "stage3_adversarial_stress", "stage4_promotion_threshold"] model_candidates.append({ @@ -152,8 +162,9 @@ def build(corpus: list[dict], candidates: list[dict], results: dict | None = Non ], "failure_modes_to_probe": ["stale_docs", "conflicting_evidence", "permission_bound", "long_context", "multilingual"], }) - # MetricFacts ONLY for real, measured results — never from seed scores. - if results is not None and cid in results: + # MetricFacts ONLY for a real measured result on a candidate that CLEARED + # Stage 0 — never from seed scores, never for a fail-closed-gated candidate. + if has_real and not gated: r = results[cid] facts.append({ "metric_fact_id": "mf.isota.%s.%s" % (cid.split(".")[-1], ts), @@ -175,17 +186,23 @@ def build(corpus: list[dict], candidates: list[dict], results: dict | None = Non "contracts": contracts, "facts": facts, "verdicts": verdicts} +def _validate(inst: dict, schema: dict) -> None: + # spec-first WITH format enforcement — e.g. MetricFact.ts "format": "date-time" + # is actually checked (needs rfc3339-validator installed to bite on date-time). + Draft202012Validator(schema, format_checker=FormatChecker()).validate(inst) + + def validate_bundle(bundle: dict) -> None: md, mc = _schema("metric-definition"), _schema("model-candidate") bc, mf = _schema("benchmark-contract"), _schema("metric-fact") for d in bundle["definitions"]: - jsonschema.validate(d, md) + _validate(d, md) for c in bundle["candidates"]: - jsonschema.validate(c, mc) + _validate(c, mc) for k in bundle["contracts"]: - jsonschema.validate(k, bc) + _validate(k, bc) for f in bundle["facts"]: - jsonschema.validate(f, mf) + _validate(f, mf) # ---- seed candidates: research-tracked models. Axis scores are ILLUSTRATIVE INPUT @@ -222,6 +239,10 @@ def main() -> int: args = ap.parse_args() corpus = json.loads(SEED_CORPUS.read_text())["items"] + # fail-fast: the corpus must conform to the vendored EvalItem schema (spec-first). + eval_item_schema = json.loads((SCHEMA_DIR / "vendored" / "EvalItem.schema.json").read_text()) + for it in corpus: + _validate(it, eval_item_schema) results = json.loads(args.results.read_text()) if args.results else None bundle = build(corpus, seed_candidates(), results=results) validate_bundle(bundle)