diff --git a/scripts/validate_v3_schemas.py b/scripts/validate_v3_schemas.py new file mode 100755 index 0000000..a6cd1f4 --- /dev/null +++ b/scripts/validate_v3_schemas.py @@ -0,0 +1,371 @@ +#!/usr/bin/env python3 +"""Validate 2000m v3 schema scaffolding and fixtures. + +The v3 foundation intentionally avoids third-party JSON Schema dependencies. +The JSON Schema files are the public contract; this script enforces enough of +Draft 2020-12 for the committed fixtures plus the v3-specific safety invariants +that schema syntax alone cannot express. +""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +from pathlib import Path +from typing import Any + +ROOT = Path(__file__).resolve().parents[1] +V3 = ROOT / "v3" +SCHEMAS = { + "2000m.v3.manifest.v1": V3 / "manifest.schema.json", + "2000m.v3.campaign.v1": V3 / "campaign.schema.json", + "2000m.v3.run-record.v1": V3 / "run-record.schema.json", + "2000m.v3.result.v1": V3 / "result.schema.json", + "2000m.v3.visual-package.v1": V3 / "visual-package.schema.json", +} +REQUIRED_SPEC_FILES = [ + V3 / "manifest.schema.json", + V3 / "campaign.schema.json", + V3 / "run-record.schema.json", + V3 / "result.schema.json", + V3 / "visual-package.schema.json", + V3 / "2000m.driver.v3.md", + V3 / "MECHANICAL_AC_SPEC.md", + V3 / "VISUAL_RUBRIC.md", + V3 / "SCORING_MODEL.md", + V3 / "SANDBOX_AND_RESOURCE_POLICY.md", +] +VALID_FIXTURES = sorted((V3 / "examples" / "valid").glob("*.json")) +INVALID_FIXTURES = { + V3 / "examples" / "invalid" / "manifest-private-path.json": "private/local ref", + V3 / "examples" / "invalid" / "campaign-framework-specific-required-field.json": "additional property", + V3 / "examples" / "invalid" / "campaign-boolean-seed.json": "integer", + V3 / "examples" / "invalid" / "visual-package-missing-capture-metadata.json": "required", + V3 / "examples" / "invalid" / "result-unsupported-claim.json": "unsupported claim", + V3 / "examples" / "invalid" / "run-record-invalid-enum.json": "enum", + V3 / "examples" / "invalid" / "result-mutated-frozen-protocol.json": "frozen protocol mutation", + V3 / "examples" / "invalid" / "result-ranked-visual-missing-evidence.json": "ranked visual result", + V3 / "examples" / "invalid" / "run-record-ranked-visual-missing-evidence.json": "ranked run-record visual status", +} + +PRIVATE_MARKERS = ( + "/Users/", + "\\Users\\", + "file://", + "~/", + "../", + "..\\", +) +OWNER_IDENTITY_MARKERS: tuple[str, ...] = () +UNSUPPORTED_CLAIM_MARKERS = ( + "makes the model smarter", + "proves workflow superiority", + "workflow superiority", + "adoption proof", + "public contender result from a fixture", + "high mechanical score means a good game", + "score equals model ranking", +) +FRAMEWORK_REQUIRED_MARKERS = ( + ".osc", + "openscaffold", + "open-scaffold", + "open scaffold", + "frameworkrequired", + "framework-specific", + "command-name credit", +) + + +class V3ValidationError(ValueError): + pass + + +def load_json(path: Path) -> Any: + try: + return json.loads(path.read_text()) + except FileNotFoundError as exc: + raise V3ValidationError(f"missing JSON file: {path}") from exc + except json.JSONDecodeError as exc: + raise V3ValidationError(f"invalid JSON in {path}: {exc}") from exc + + +def require(condition: bool, message: str) -> None: + if not condition: + raise V3ValidationError(message) + + +def is_plain_int(value: Any) -> bool: + return isinstance(value, int) and not isinstance(value, bool) + + +def is_number(value: Any) -> bool: + return (isinstance(value, int) or isinstance(value, float)) and not isinstance(value, bool) + + +def resolve_ref(schema: dict[str, Any], root: dict[str, Any]) -> dict[str, Any]: + ref = schema.get("$ref") + if not isinstance(ref, str): + return schema + require(ref.startswith("#/$defs/"), f"only local $defs refs are supported by validator, got {ref}") + name = ref.rsplit("/", 1)[-1] + defs = root.get("$defs", {}) + require(name in defs, f"unresolved $ref {ref}") + target = defs[name] + require(isinstance(target, dict), f"$ref target {ref} is not a schema object") + return target + + +def validate_json_schema(instance: Any, schema: dict[str, Any], *, root: dict[str, Any] | None = None, path: str = "$", seen_refs: int = 0) -> None: + root = root or schema + if "$ref" in schema: + require(seen_refs < 20, f"{path}: excessive $ref recursion") + return validate_json_schema(instance, resolve_ref(schema, root), root=root, path=path, seen_refs=seen_refs + 1) + + if "const" in schema: + require(instance == schema["const"], f"{path}: expected const {schema['const']!r}, got {instance!r}") + if "enum" in schema: + require(instance in schema["enum"], f"{path}: value {instance!r} not in enum {schema['enum']!r}") + + schema_type = schema.get("type") + if isinstance(schema_type, list): + if any(_matches_type(instance, item) for item in schema_type): + pass + else: + raise V3ValidationError(f"{path}: expected one of types {schema_type!r}, got {type(instance).__name__}") + elif isinstance(schema_type, str): + require(_matches_type(instance, schema_type), f"{path}: expected type {schema_type}, got {type(instance).__name__}") + + if isinstance(instance, str): + if "minLength" in schema: + require(len(instance) >= schema["minLength"], f"{path}: string shorter than minLength {schema['minLength']}") + if "pattern" in schema: + require(re.search(schema["pattern"], instance), f"{path}: string {instance!r} does not match pattern {schema['pattern']!r}") + + if is_number(instance): + if "minimum" in schema: + require(instance >= schema["minimum"], f"{path}: value {instance!r} below minimum {schema['minimum']!r}") + if "maximum" in schema: + require(instance <= schema["maximum"], f"{path}: value {instance!r} above maximum {schema['maximum']!r}") + + if isinstance(instance, list): + if "minItems" in schema: + require(len(instance) >= schema["minItems"], f"{path}: array has fewer than minItems {schema['minItems']}") + if "maxItems" in schema: + require(len(instance) <= schema["maxItems"], f"{path}: array has more than maxItems {schema['maxItems']}") + if schema.get("uniqueItems"): + encoded = [json.dumps(item, sort_keys=True) for item in instance] + require(len(encoded) == len(set(encoded)), f"{path}: array items are not unique") + if "items" in schema: + items_schema = schema["items"] + require(isinstance(items_schema, dict), f"{path}: validator supports only object items schemas") + for idx, item in enumerate(instance): + validate_json_schema(item, items_schema, root=root, path=f"{path}[{idx}]", seen_refs=seen_refs) + + if isinstance(instance, dict): + required = schema.get("required", []) + require(isinstance(required, list), f"{path}: required must be a list") + for key in required: + require(key in instance, f"{path}: missing required property {key!r}") + properties = schema.get("properties", {}) + require(isinstance(properties, dict), f"{path}: properties must be object") + if schema.get("additionalProperties") is False: + allowed = set(properties) + extra = sorted(set(instance) - allowed) + if extra: + raise V3ValidationError(f"{path}: additional property {extra[0]!r} is not allowed") + for key, value in instance.items(): + if key in properties: + prop_schema = properties[key] + require(isinstance(prop_schema, dict), f"{path}.{key}: property schema must be object") + validate_json_schema(value, prop_schema, root=root, path=f"{path}.{key}", seen_refs=seen_refs) + + +def _matches_type(value: Any, schema_type: str) -> bool: + if schema_type == "object": + return isinstance(value, dict) + if schema_type == "array": + return isinstance(value, list) + if schema_type == "string": + return isinstance(value, str) + if schema_type == "integer": + return is_plain_int(value) + if schema_type == "number": + return is_number(value) + if schema_type == "boolean": + return isinstance(value, bool) + if schema_type == "null": + return value is None + raise V3ValidationError(f"unsupported schema type {schema_type!r}") + + +def iter_strings(value: Any, path: str = "$", *, include_keys: bool = False): + if isinstance(value, str): + yield path, value + elif isinstance(value, list): + for idx, item in enumerate(value): + yield from iter_strings(item, f"{path}[{idx}]", include_keys=include_keys) + elif isinstance(value, dict): + for key, item in value.items(): + if include_keys: + yield f"{path}.{key}.__key__", str(key) + yield from iter_strings(item, f"{path}.{key}", include_keys=include_keys) + + +def check_public_safe_strings(value: Any, *, allow_banned_claim_list: bool = False) -> None: + for path, text in iter_strings(value): + stripped = text.strip() + lowered = stripped.lower() + if path.endswith(".bannedClaims") or ".bannedClaims[" in path: + if allow_banned_claim_list: + continue + require(not stripped.startswith("/"), f"{path}: private/local ref starts with absolute slash: {text!r}") + require(not stripped.startswith("~"), f"{path}: private/local ref starts with home marker: {text!r}") + require(not re.match(r"^[A-Za-z]:", stripped), f"{path}: private/local ref uses Windows drive path: {text!r}") + require(not stripped.startswith("\\\\"), f"{path}: private/local ref uses UNC path: {text!r}") + for marker in PRIVATE_MARKERS: + require(marker.lower() not in lowered, f"{path}: private/local ref marker {marker!r}: {text!r}") + for marker in OWNER_IDENTITY_MARKERS: + require(marker.lower() not in lowered, f"{path}: owner identity marker {marker!r}: {text!r}") + # Discord snowflakes are typically 17-20 digits. Keep a broad guard for public records. + require(not bool(re.search(r"\b\d{17,20}\b", stripped)), f"{path}: possible Discord ID: {text!r}") + + +def check_unsupported_claims(value: Any) -> None: + for path, text in iter_strings(value): + if path.endswith(".bannedClaims") or ".bannedClaims[" in path: + continue + lowered = text.lower() + for marker in UNSUPPORTED_CLAIM_MARKERS: + require(marker not in lowered, f"{path}: unsupported claim marker {marker!r}") + + +def check_schema_neutrality(schema: dict[str, Any], path: Path) -> None: + for string_path, text in iter_strings(schema, include_keys=True): + lowered = text.lower() + if string_path.endswith(".__key__") or ".required[" in string_path: + for marker in FRAMEWORK_REQUIRED_MARKERS: + require(marker not in lowered, f"{path}:{string_path}: framework-specific schema field marker {marker!r}") + # Schema files must not contain native controller/Lane C implementation fields. + require("controllerstatus" not in lowered, f"{path}:{string_path}: controllerStatus belongs to v2, not v3 foundation") + + +def validate_semantics(data: dict[str, Any], schema_version: str) -> None: + check_public_safe_strings(data, allow_banned_claim_list=True) + check_unsupported_claims(data) + + if schema_version == "2000m.v3.campaign.v1": + freeze = data["protocolFreeze"] + require(freeze["freezeBeforeLiveResults"] is True, "campaign protocol must freeze before live results") + require(freeze["noProtocolMutationAfterLiveResults"] is True, "campaign must forbid protocol mutation after live results") + for seed in freeze["visualSeeds"]: + require(is_plain_int(seed), "campaign visualSeeds must contain integers, not booleans") + seen_lanes: set[str] = set() + enabled_lanes = {lane["laneId"] for lane in data["lanes"] if lane["enabled"]} + for lane in data["lanes"]: + lane_id = lane["laneId"] + require(lane_id not in seen_lanes, f"duplicate laneId {lane_id}") + seen_lanes.add(lane_id) + require(lane_id != "C", "Lane C/controller behavior is out of scope for the v3 foundation slice") + require("controller" not in json.dumps(lane).lower(), "controller behavior is out of scope for the v3 foundation slice") + require(lane["processType"] in {"naked-model", "workflow-system", "scripted-agent", "human-operated", "other"}, "unsupported processType") + for pair in data["pairs"]: + require(is_plain_int(pair["taskSeed"]), "campaign pair taskSeed must be integer, not boolean") + require(set(pair["enabledLanes"]).issubset(enabled_lanes), f"pair {pair['pairId']} references disabled or missing lane") + elif schema_version == "2000m.v3.run-record.v1": + freeze = data["protocolFreeze"] + require(freeze["frozen"] is True, "run record must point at a frozen campaign/protocol") + require(freeze["changedAfterLiveResults"] is False, "frozen protocol mutation is calibration-only and invalid in foundation fixtures") + require(freeze["scorerMutationObserved"] is False, "scorer mutation is invalid in foundation fixtures") + require(data["entrant"]["processType"] in {"naked-model", "workflow-system", "scripted-agent", "human-operated", "other"}, "unsupported processType") + visual = data["visual"] + if visual["ranked"] is True or visual["blockReason"] == "none": + require(visual["ranked"] is True, "ranked run-record visual status must set visual.ranked=true when blockReason is none") + require(visual["blockReason"] == "none", "ranked run-record visual status must use blockReason none") + require(bool(visual["visualPackageRef"].strip()), "ranked run-record visual status requires non-empty visualPackageRef") + require(bool(visual.get("captureCommandResultRef", "").strip()), "ranked run-record visual status requires non-empty captureCommandResultRef") + elif schema_version == "2000m.v3.result.v1": + freeze = data["protocolFreeze"] + require(freeze["changedAfterLiveResults"] is False, "frozen protocol mutation is calibration-only and invalid in foundation fixtures") + require(freeze["scorerMutationObserved"] is False, "frozen protocol mutation is calibration-only and invalid in foundation fixtures") + require(freeze["calibrationOnlyIfChanged"] is True, "changed protocol must force calibration-only handling") + require(data["claimBoundary"] != "public-benchmark-support", "foundation fixtures must not claim public benchmark support") + require(data["evidence"]["claimBoundary"] == data["claimBoundary"], "evidence claimBoundary must match result claimBoundary") + visual = data["visual"] + if visual["ranked"] is True or visual["blockReason"] == "none": + require(visual["ranked"] is True, "ranked visual result must set visual.ranked=true when blockReason is none") + require(visual["blockReason"] == "none", "ranked visual result must use blockReason none") + require(bool(visual["visualPackageRef"].strip()), "ranked visual result requires non-empty visualPackageRef") + require(bool(visual["rubricRecordRef"].strip()), "ranked visual result requires non-empty rubricRecordRef") + require(visual["captureDeterminism"] == "passed", "ranked visual result requires captureDeterminism passed") + elif schema_version == "2000m.v3.visual-package.v1": + require(data["anonymized"] is True, "visual package must be anonymized before blind review") + require(data["mappingSealedBeforeReview"] is True, "blind label map must be sealed before review") + for window in data["windows"]: + for key in ("seed", "captureCommand", "screenshotRef", "replayRef", "frameMetadataRef", "rubricMetadataRef", "fps", "frameCount", "inputSequenceRef", "stateChecksum", "frameChecksum"): + require(key in window, f"visual package missing capture metadata {key}") + require(is_plain_int(window["seed"]), "capture seed must be integer, not boolean") + elif schema_version == "2000m.v3.manifest.v1": + require(data["protocolVersion"] == "2000m.driver.v3", "manifest protocolVersion must be 2000m.driver.v3") + if "capture" not in data or "playable" not in data: + # Allowed for mechanical-only manifests, but visual/product rank must be blocked elsewhere. + pass + + +def validate_fixture(path: Path) -> None: + data = load_json(path) + require(isinstance(data, dict), f"{path}: top-level JSON must be object") + schema_version = data.get("schemaVersion") + require(isinstance(schema_version, str), f"{path}: missing schemaVersion") + require(schema_version in SCHEMAS, f"{path}: unsupported v3 schemaVersion {schema_version!r}") + schema = load_json(SCHEMAS[schema_version]) + require(isinstance(schema, dict), f"{SCHEMAS[schema_version]}: schema must be object") + validate_json_schema(data, schema) + validate_semantics(data, schema_version) + + +def validate_all() -> None: + for path in REQUIRED_SPEC_FILES: + require(path.exists(), f"missing required v3 foundation file: {path.relative_to(ROOT)}") + for path in SCHEMAS.values(): + schema = load_json(path) + require(isinstance(schema, dict), f"{path}: schema must be object") + require(schema.get("$schema") == "https://json-schema.org/draft/2020-12/schema", f"{path}: schema draft marker must be 2020-12") + check_schema_neutrality(schema, path.relative_to(ROOT)) + require(bool(VALID_FIXTURES), "no v3 valid fixtures found") + for path in VALID_FIXTURES: + validate_fixture(path) + print(f"OK valid: {path.relative_to(ROOT)}") + for path, expected in INVALID_FIXTURES.items(): + try: + validate_fixture(path) + except V3ValidationError as exc: + message = str(exc).lower() + require(expected.lower() in message, f"{path.relative_to(ROOT)} failed for unexpected reason: {exc}; expected marker {expected!r}") + print(f"OK invalid: {path.relative_to(ROOT)} -> {exc}") + else: + raise V3ValidationError(f"invalid fixture unexpectedly passed: {path.relative_to(ROOT)}") + + +def main() -> int: + parser = argparse.ArgumentParser(description="Validate v3 schemas, examples, and semantic guard fixtures") + parser.add_argument("paths", nargs="*", help="Specific v3 JSON fixture paths to validate; defaults to full suite") + args = parser.parse_args() + try: + if args.paths: + for raw in args.paths: + path = Path(raw) + validate_fixture(path if path.is_absolute() else ROOT / path) + print(f"OK: {raw}") + else: + validate_all() + except V3ValidationError as exc: + print(f"FAIL: {exc}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/v3/2000m.driver.v3.md b/v3/2000m.driver.v3.md new file mode 100644 index 0000000..eb02701 --- /dev/null +++ b/v3/2000m.driver.v3.md @@ -0,0 +1,112 @@ +# 2000m driver protocol v3 draft + +Status: executable-spec draft for the v3 foundation slice. This document defines the target protocol shape for future v3 scorer work; it does not implement a scorer and does not change v0/v1/v2 semantics. + +## Goals + +The v3 driver keeps deterministic JSON-line scoring as the mechanical substrate while adding enough protocol detail to support replay, hidden challenges, invalid-enum rejection, and visual-capture trust. + +A submitted artifact declares the protocol through `v3/manifest.schema.json` with: + +```json +{ + "schemaVersion": "2000m.v3.manifest.v1", + "protocolVersion": "2000m.driver.v3", + "driver": { "command": "...", "args": [] } +} +``` + +The mechanical scorer talks only to the declared driver subprocess over stdin/stdout. Capture and playable surfaces are separate manifest sections and must not change mechanical pass counts. + +## Line format + +Each request and response is one UTF-8 JSON object per line. + +Requests use: + +```json +{ + "protocolVersion": "2000m.driver.v3", + "requestId": "req-0001", + "command": "init", + "payload": {} +} +``` + +Responses use: + +```json +{ + "protocolVersion": "2000m.driver.v3", + "requestId": "req-0001", + "ok": true, + "payload": {} +} +``` + +Error responses use: + +```json +{ + "protocolVersion": "2000m.driver.v3", + "requestId": "req-0001", + "ok": false, + "error": { + "code": "invalid_request", + "message": "short public-safe message" + } +} +``` + +Allowed error codes: `invalid_request`, `invalid_state`, `unsupported_command`, `schema_violation`, `challenge_unavailable`, `internal_error`. + +## Commands + +| Command | Purpose | Required payload fields | Required response payload | +|---|---|---|---| +| `init` | Start a deterministic run. | `seed`, `config` | `state`, `stateChecksum`, `capabilities` | +| `step` | Advance one tick with player input. | `input` | `state`, `events`, `stateChecksum` | +| `state` | Return current state without advancing. | none | `state`, `stateChecksum` | +| `reset` | Reset to a seed/config. | `seed`, `config` | `state`, `stateChecksum` | +| `profile` | Return driver-reported diagnostics. | none | `profile` | +| `replay` | Re-run an input stream for checksum comparison. | `seed`, `config`, `inputs` | `finalState`, `stateChecksums`, `replayChecksum` | +| `challenge` | Run a public or hidden challenge fixture. | `challengeId`, `seed`, `inputs` | `state`, `events`, `stateChecksum`, `challengeResult` | + +`config` must be public and scorer-controlled. Entrants must not require private setup flags or lane-specific hints. + +## GameState draft + +Every `state` field must include: + +| Field | Type | Notes | +|---|---:|---| +| `tick` | integer | Monotonic, starts at 0 after `init`/`reset`. | +| `seed` | integer | Plain integer; booleans are invalid. | +| `skier` | object | Position, velocity, airborne/style/crashed state. | +| `world` | object | Slope/window metadata and obstacle list. | +| `monster` | object or null | Spawn/pursuit/contact/flee state when applicable. | +| `score` | object | Distance, style, crash count, rank-neutral counters. | +| `events` | array | Event labels from the enum below. | + +Allowed obstacle types: `tree`, `bigtree`, `stump`, `mogul`, `rock`, `ramp`. + +Allowed event labels: `spawn`, `turn`, `collision`, `crash`, `recover`, `ramp_enter`, `airborne`, `land`, `style_gain`, `style_loss`, `monster_spawn`, `monster_pursuit`, `monster_contact`, `monster_flee`. + +Any unsupported enum value is a schema violation. The future scorer may fail or rank-block the affected mechanical result, but it must not silently coerce unknown values. + +## Checksums + +`stateChecksum` is `sha256:` plus the lowercase SHA-256 digest of canonical JSON for the emitted state. Canonical JSON means sorted keys, UTF-8, no insignificant whitespace, and stable number serialization as defined by the scorer implementation. + +`replayChecksum` is the SHA-256 digest over the ordered list of `stateChecksum` values from replay execution. + +## Hidden/public challenges + +Campaign files must declare whether a challenge set is public, hidden, or calibration-only before live results. Hidden challenge IDs may be withheld from entrants, but the policy and denominator impact must be frozen in `v3/campaign.schema.json` records. + +## Non-goals in this draft + +- No native Lane C/controller behavior. +- No framework-specific required fields. +- No score credit for workflow command names. +- No visual/product score from mechanical driver state alone. diff --git a/v3/MECHANICAL_AC_SPEC.md b/v3/MECHANICAL_AC_SPEC.md new file mode 100644 index 0000000..7337ecb --- /dev/null +++ b/v3/MECHANICAL_AC_SPEC.md @@ -0,0 +1,54 @@ +# 2000m v3 mechanical AC table draft + +Status: executable-spec draft. This table defines the first concrete Track 1 acceptance-criteria shape for future scorer implementation. It is not yet a scorer and does not change v0/v1/v2 results. + +## Scoring boundary + +Mechanical correctness answers only whether the artifact obeys the deterministic game-logic driver contract. It does not decide whether the game looks good, is fun, or demonstrates workflow value. + +Draft denominator: 24 binary ACs plus diagnostic quality fields. Hidden challenges may be used only if their policy is frozen before live results. + +## AC table + +| ID | Area | Public fixture | Hidden fixture | Pass threshold | Notes | +|---|---|---:|---:|---|---| +| M01 | Manifest/protocol | yes | no | manifest validates as `2000m.v3.manifest.v1`; driver declares `2000m.driver.v3` | Capture/playable may block visual rank but not mechanical. | +| M02 | Init determinism | yes | yes | same seed/config emits byte-stable canonical state checksum | Booleans are invalid seeds. | +| M03 | Reset determinism | yes | yes | reset returns to the same state as init for the same seed/config | | +| M04 | Step determinism | yes | yes | same input stream emits identical state/event/checksum sequence | | +| M05 | State idempotence | yes | no | repeated `state` calls do not advance ticks | | +| M06 | Schema validity | yes | yes | every emitted state validates; unsupported enum values fail | Covers obstacle and event poison. | +| M07 | Obstacle generation | yes | yes | fixed seeds produce stable obstacle fields with allowed types | | +| M08 | Skier movement | yes | no | left/right/neutral input changes position and velocity within bounds | | +| M09 | Collision correctness | yes | yes | collisions with solid obstacles crash or recover as specified | | +| M10 | Recovery behavior | yes | no | crashed skier can recover through legal steps | | +| M11 | Ramp entry | yes | yes | ramp collision enters airborne state rather than ordinary crash | | +| M12 | Airborne/landing | yes | yes | airborne duration and landing transitions are deterministic | | +| M13 | Style scoring | yes | no | style gain/loss events update score consistently | | +| M14 | Monster spawn | yes | yes | monster appears under frozen trigger conditions | | +| M15 | Monster pursuit | yes | yes | monster moves toward skier under pressure window | | +| M16 | Monster contact/flee | yes | yes | contact/flee events are emitted and deterministic | | +| M17 | Replay checksum | yes | yes | `replay` reproduces final state and checksum list | | +| M18 | Public challenge | yes | no | public challenge set can be executed by ID | | +| M19 | Hidden challenge isolation | no | yes | hidden challenge inputs are not required in entrant manifest | Policy must be predeclared. | +| M20 | Regression stability | yes | yes | final artifact reruns pass the same AC set twice | | +| M21 | Error semantics | yes | no | invalid requests return structured errors, not malformed output | | +| M22 | No scorer mutation | yes | yes | result record shows frozen scorer/protocol were not mutated | Schema-level guard only in this PR. | +| M23 | No private setup hints | yes | yes | driver passes with public config only | | +| M24 | Mechanical/visual separation | yes | no | missing native capture does not alter mechanical pass count | Visual rank blocker is recorded separately. | + +## Draft composite + +If a mechanical convenience composite is rendered later, it must be labeled `mechanical.compositeScore` and computed only from mechanical ACs. It must not include visual taste, workflow resilience, evidence volume, generations-to-finish, or framework usage. + +Draft formula for future implementation: + +```text +mechanical.compositeScore = 100 * passed_mechanical_acs / total_ranked_mechanical_acs +``` + +Skipped or untestable ranked ACs count as zero unless the frozen campaign declares the AC as probe-only before live results. + +## Invalid data handling + +Unsupported obstacle/event enums, malformed state, boolean seeds, private setup dependencies, and scorer/protocol mutation are hard mechanical failures or rank blockers. The future scorer must report the exact reason rather than silently repairing entrant output. diff --git a/v3/README.md b/v3/README.md index 6c223d7..fe8f1c7 100644 --- a/v3/README.md +++ b/v3/README.md @@ -24,6 +24,12 @@ The benchmark remains independent and workflow-agnostic. It may test failure mod - [`EVIDENCE_AND_CLAIMS.md`](EVIDENCE_AND_CLAIMS.md) — evidence/governance requirements, claim ladder, and anti-rigging guardrails. - [`IMPLEMENTATION_PLAN.md`](IMPLEMENTATION_PLAN.md) — staged path from this design packet to v3 scorer/harness implementation. - [`SPEC_GAPS_AND_DECISIONS.md`](SPEC_GAPS_AND_DECISIONS.md) — OMX advisory review gaps to resolve before executable v3 scorer work. +- [`2000m.driver.v3.md`](2000m.driver.v3.md) — draft exact v3 driver protocol. +- [`MECHANICAL_AC_SPEC.md`](MECHANICAL_AC_SPEC.md) — draft mechanical acceptance-criteria table. +- [`VISUAL_RUBRIC.md`](VISUAL_RUBRIC.md) — draft native capture and blind-review rubric. +- [`SCORING_MODEL.md`](SCORING_MODEL.md) — draft separated-track scoring and blocker model. +- [`SANDBOX_AND_RESOURCE_POLICY.md`](SANDBOX_AND_RESOURCE_POLICY.md) — draft scorer sandbox/resource policy. +- `*.schema.json` plus `examples/valid/` and `examples/invalid/` — v3 schema foundation fixtures validated by `scripts/validate_v3_schemas.py`. ## Claim boundary diff --git a/v3/SANDBOX_AND_RESOURCE_POLICY.md b/v3/SANDBOX_AND_RESOURCE_POLICY.md new file mode 100644 index 0000000..343f6ec --- /dev/null +++ b/v3/SANDBOX_AND_RESOURCE_POLICY.md @@ -0,0 +1,65 @@ +# 2000m v3 sandbox and resource policy draft + +Status: executable-spec draft. This policy defines expected scorer/harness execution boundaries for future v3 implementation. It does not run a live pilot. + +## Purpose + +v3 needs deterministic, fair, and public-safe execution. Resource policy is part of the frozen campaign contract so entrants cannot gain hidden advantage through environment differences. + +## Draft default policy + +| Surface | Default | +|---|---| +| Mechanical driver timeout | 30 seconds per scorer command batch unless campaign overrides before freeze. | +| Capture timeout | 60 seconds per seed/window capture unless campaign overrides before freeze. | +| Build timeout | 5 minutes per artifact build in calibration smokes. | +| Network during scoring | Disabled. | +| Network during pre-run setup | Only allowlisted package installation before protocol freeze. | +| Filesystem | Scorer reads artifact root and benchmark fixtures; public records must use repo-relative refs. | +| Build cache | Allowed only if cache policy is identical across lanes and declared. | +| Stdout/stderr limits | Capture compact refs; raw logs remain private unless scanned and explicitly allowed. | +| Secrets | No secrets required for scoring or capture. | + +## Command failure handling + +Every failed command record must include: + +- command label; +- public-safe command string or argv; +- exit code or timeout marker; +- stdout/stderr refs or compact redacted summary; +- whether the failure blocks mechanical, visual, workflow, or evidence track ranking. + +A command failure must not be silently converted into a pass, and it must not trigger scorer/rubric changes after live output inspection unless the affected run becomes calibration-only. + +## Path policy + +Public records must reject: + +- local absolute paths; +- home-directory paths; +- Windows drive/UNC local paths; +- `file://` URLs; +- traversal refs such as `../`; +- private repo paths; +- owner identity or private Discord IDs. + +Use repo-relative refs or public URLs only. + +## Lane fairness + +For paired campaigns, both lanes receive the same resource policy: + +- same model/runtime label; +- same generation cap; +- same prompt and feedback budget; +- same exact scorer diagnostics; +- same context-wipe phase; +- same visual seeds/windows; +- same reviewer budget; +- same network/cache policy; +- explicit disclosure of any human intervention. + +## Future implementation notes + +The first v3 scorer implementation should start with sandbox policy validation and fixture-based smokes. It should not run a live contender campaign until schemas, scenarios, visual package validation, and separated result rendering are in place. diff --git a/v3/SCORING_MODEL.md b/v3/SCORING_MODEL.md new file mode 100644 index 0000000..3d9a5fb --- /dev/null +++ b/v3/SCORING_MODEL.md @@ -0,0 +1,84 @@ +# 2000m v3 scoring model draft + +Status: executable-spec draft. This model defines separated tracks, blockers, and claim boundaries for future implementation. It does not run a new evidence campaign. + +## Track separation + +v3 reports four tracks separately: + +1. `mechanical` — deterministic game-logic correctness. +2. `visual` — native visual/playable artifact quality. +3. `workflow` — recovery, feedback routing, regression protection, impossible/stale handling, and handoff. +4. `evidence` — replayability, public safety, compact record quality, and claim-boundary hygiene. + +No single composite may hide these tracks. If a UI later renders a convenience summary, it must preserve every component and state that workflow score is not model intelligence. + +## Draft track fields + +### Mechanical + +- `mechanical.ranked` +- `mechanical.protocolVersion` +- `mechanical.passCount` +- `mechanical.totalAcs` +- `mechanical.compositeScore` +- `mechanical.determinism.pass` +- `mechanical.failedAcs[]` +- `mechanical.hiddenChallengeSummary` +- `mechanical.regressionSummary` +- `mechanical.resultJsonRef` + +### Visual + +- `visual.ranked` +- `visual.blockReason` +- `visual.visualPackageRef` +- `visual.captureDeterminism` +- `visual.rubricRecordRef` +- optional `visual.score` + +### Workflow + +- `workflow.contextWipeRecoveryScore` +- `workflow.feedbackDecisionScore` +- `workflow.regressionProtectionScore` +- `workflow.impossibleRequirementHandlingScore` +- `workflow.handoffScore` +- `workflow.finalRecommendation` +- `workflow.rationaleRefs[]` + +### Evidence + +- `evidence.replayable` +- `evidence.publicSafe` +- `evidence.privateRefsBlocked` +- `evidence.compactSummaryRef` +- `evidence.requiredRefsMissing[]` +- `evidence.claimBoundary` + +## Blockers + +| Blocker | Mechanical | Visual | Workflow | Evidence/claim impact | +|---|---|---|---|---| +| Missing native capture | unaffected | rank-blocked | unaffected unless scenario requires handling it | visual claims forbidden | +| Invalid driver enum/schema | failed/rank-blocked | unaffected | may affect feedback handling | exact reason required | +| Private/local path in public record | unaffected unless manifest depends on it | rank-blocked if visual ref | evidence fails | public claims blocked | +| Frozen protocol changed after live results | affected run calibration-only | affected run calibration-only | affected run calibration-only | public support blocked | +| Unsupported claim text | unaffected | unaffected | unaffected | evidence fails; public claims blocked | +| Evidence volume without decision quality | unaffected | unaffected | no score credit | evidence may still pass only if compact and useful | + +## Claim ceiling + +| `claimBoundary` | Meaning | +|---|---| +| `calibration-only` | Fixture, smoke, or changed-protocol result; not contender evidence. | +| `no-support` | Valid campaign found no support for the tested claim. | +| `directional-signal` | Pattern worth more study; causality unproven. | +| `repeatable-workflow-value-support-candidate` | Predeclared thresholds met across valid pairs. | +| `public-benchmark-support` | Reserved for larger public-safe campaigns with independent review. | + +Foundation fixtures in this PR must stay `calibration-only`. + +## Anti-paperwork rule + +Workflow and evidence tracks must not award score for evidence volume, framework names, command names, or directory shape. Score only generic recoverability, correctness of decisions, feedback parity, rerun evidence, and public-safe replayability. diff --git a/v3/SPEC_GAPS_AND_DECISIONS.md b/v3/SPEC_GAPS_AND_DECISIONS.md index 1ae70d8..714390e 100644 --- a/v3/SPEC_GAPS_AND_DECISIONS.md +++ b/v3/SPEC_GAPS_AND_DECISIONS.md @@ -119,7 +119,7 @@ Needed before public results: ## Next design actions -Before implementation, add or refine: +The foundation slice adds draft versions of these artifacts plus schema fixtures. Before scorer behavior changes, refine them against review feedback and keep the protocol frozen for any campaign that uses them: 1. `v3/2000m.driver.v3.md` — exact driver protocol. 2. `v3/MECHANICAL_AC_SPEC.md` — concrete mechanical AC table. diff --git a/v3/VISUAL_RUBRIC.md b/v3/VISUAL_RUBRIC.md new file mode 100644 index 0000000..f9d5261 --- /dev/null +++ b/v3/VISUAL_RUBRIC.md @@ -0,0 +1,72 @@ +# 2000m v3 visual/product rubric draft + +Status: executable-spec draft. This rubric defines visual/product evidence expectations and blockers; it does not create public proof or run a pilot. + +## Visual track boundary + +Visual/product quality answers whether the entrant produced a coherent visible/playable SkiFree-inspired artifact. It is separate from mechanical correctness and workflow resilience. + +A missing native capture or playable surface sets: + +```text +visual.ranked = false +visual.blockReason = missing-native-capture-or-playable-surface +``` + +A neutral proxy renderer may support private inspection but cannot substitute for submitted native capture in public visual ranking. + +## Required package + +Every ranked visual package must validate against `v3/visual-package.schema.json` and include: + +- blind contact sheet; +- sealed blind label map; +- fixed-seed screenshots; +- deterministic GIF or WebM/replay artifacts; +- per-window frame metadata; +- rubric record; +- artifact digest captured before rendering; +- capture command and checksums for each window. + +## Capture windows + +| Window | Required evidence | Purpose | +|---|---|---| +| `early-game` | screenshot + replay + frame metadata | Basic skier/slope/readability. | +| `mid-run-obstacle-field` | screenshot + replay + frame metadata | Obstacle density without unreadable clutter. | +| `ramp-style-sequence` | screenshot + replay + frame metadata | Ramps, airborne/landing, style readability. | +| `monster-pressure` | screenshot + replay + frame metadata | Monster appearance and threat readability. | +| `post-feedback-rerun` | screenshot + replay + frame metadata | Visual regression/improvement after feedback. | + +Seeds and windows must be frozen in the campaign before live outputs are seen. + +## Draft rubric dimensions + +Each dimension is scored 0-5 by reviewer or deterministic diagnostic where stated. Do not use these numbers to rescue a mechanically invalid artifact. + +| Dimension | Weight | Evidence | 0 means | 5 means | +|---|---:|---|---|---| +| Skier readability | 20% | screenshots/replay | skier cannot be found | skier is consistently clear | +| Slope and motion clarity | 15% | replay/frames | motion unreadable or static | motion reads as downhill skiing | +| Obstacle readability | 15% | screenshots/frames | clutter/objects impossible to parse | obstacles are distinct and navigable | +| Ramp/style communication | 10% | ramp window replay | ramp/style invisible | state changes are visually obvious | +| Monster pressure | 10% | monster window | monster absent/unreadable | monster reads as active threat | +| Visual coherence | 15% | contact sheet | inconsistent/noisy/non-game look | coherent homage style with original assets | +| Capture determinism | 10% | checksums | repeated capture differs | repeated capture matches | +| Playable surface | 5% | playable command/ref | no playable surface | surface launches from documented command/ref | + +## Blocking rules + +The visual track is rank-blocked when: + +- capture command is missing or fails without valid rerun; +- screenshot/replay/frame metadata refs are missing; +- frame metadata omits seed, window, frame count, FPS, input ref, state checksum, or frame checksum; +- fixed seeds/windows changed after live result inspection; +- blind label map was opened before rating; +- package/public record contains private local refs; +- asset attestation fails or copied/extracted SkiFree assets are found. + +## Owner taste vs public preference + +Owner taste may be recorded as a separate note. It is not a public human-preference score unless the campaign predeclares reviewer count, blinding, form, weights, and tie handling. diff --git a/v3/campaign.schema.json b/v3/campaign.schema.json new file mode 100644 index 0000000..6d97467 --- /dev/null +++ b/v3/campaign.schema.json @@ -0,0 +1,359 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/graphanov/2000m/v3/campaign.schema.json", + "title": "2000m v3 campaign freeze contract", + "type": "object", + "additionalProperties": false, + "required": [ + "schemaVersion", + "campaignId", + "title", + "status", + "protocolFreeze", + "controls", + "lanes", + "pairs", + "claimBoundary", + "bannedClaims" + ], + "properties": { + "schemaVersion": { + "const": "2000m.v3.campaign.v1" + }, + "campaignId": { + "type": "string", + "pattern": "^[a-z0-9][a-z0-9-]{2,100}$" + }, + "title": { + "type": "string", + "minLength": 1 + }, + "status": { + "type": "string", + "enum": [ + "draft", + "frozen", + "calibration", + "pilot", + "complete" + ] + }, + "protocolFreeze": { + "type": "object", + "additionalProperties": false, + "required": [ + "freezeBeforeLiveResults", + "noProtocolMutationAfterLiveResults", + "mutationAfterLiveResultsPolicy", + "benchmarkCommit", + "driverProtocolRef", + "mechanicalAcSpecRef", + "visualRubricRef", + "scoringModelRef", + "sandboxPolicyRef", + "scenarioRefs", + "feedbackPacketRef", + "decisionRulesRef", + "captureWindows", + "visualSeeds", + "hiddenChallengePolicy" + ], + "properties": { + "freezeBeforeLiveResults": { + "type": "boolean" + }, + "noProtocolMutationAfterLiveResults": { + "type": "boolean" + }, + "mutationAfterLiveResultsPolicy": { + "type": "string", + "enum": [ + "affected-runs-calibration-only" + ] + }, + "benchmarkCommit": { + "type": "string", + "minLength": 7 + }, + "driverProtocolRef": { + "type": "string", + "minLength": 1 + }, + "mechanicalAcSpecRef": { + "type": "string", + "minLength": 1 + }, + "visualRubricRef": { + "type": "string", + "minLength": 1 + }, + "scoringModelRef": { + "type": "string", + "minLength": 1 + }, + "sandboxPolicyRef": { + "type": "string", + "minLength": 1 + }, + "scenarioRefs": { + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "minLength": 1 + } + }, + "feedbackPacketRef": { + "type": "string", + "minLength": 1 + }, + "decisionRulesRef": { + "type": "string", + "minLength": 1 + }, + "captureWindows": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "type": "string", + "enum": [ + "early-game", + "mid-run-obstacle-field", + "ramp-style-sequence", + "monster-pressure", + "post-feedback-rerun" + ] + } + }, + "visualSeeds": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "type": "integer" + } + }, + "hiddenChallengePolicy": { + "type": "string", + "enum": [ + "declared-public-hidden-split", + "public-only-calibration", + "none" + ] + } + } + }, + "controls": { + "type": "object", + "additionalProperties": false, + "required": [ + "model", + "runtime", + "generationCap", + "promptBudget", + "scorerFeedbackBudget", + "reviewerFeedbackBudget", + "contextWipePhase", + "samePromptBudget", + "sameFeedbackPackets", + "sameScorerDiagnostics", + "sameVisualSeeds", + "sameReviewerBudget", + "wallClockPolicy", + "networkPolicy" + ], + "properties": { + "model": { + "type": "string", + "minLength": 1 + }, + "runtime": { + "type": "string", + "minLength": 1 + }, + "generationCap": { + "type": "integer", + "minimum": 1 + }, + "promptBudget": { + "type": "integer", + "minimum": 1 + }, + "scorerFeedbackBudget": { + "type": "integer", + "minimum": 0 + }, + "reviewerFeedbackBudget": { + "type": "integer", + "minimum": 0 + }, + "contextWipePhase": { + "type": "string", + "enum": [ + "after-first-scored-generation", + "disabled-for-calibration" + ] + }, + "samePromptBudget": { + "type": "boolean" + }, + "sameFeedbackPackets": { + "type": "boolean" + }, + "sameScorerDiagnostics": { + "type": "boolean" + }, + "sameVisualSeeds": { + "type": "boolean" + }, + "sameReviewerBudget": { + "type": "boolean" + }, + "wallClockPolicy": { + "type": "string", + "enum": [ + "diagnostic-only", + "controlled-budget", + "not-used" + ] + }, + "networkPolicy": { + "type": "string", + "enum": [ + "disabled-during-scoring", + "allowlisted-before-freeze", + "not-used" + ] + } + } + }, + "lanes": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/$defs/lane" + } + }, + "pairs": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/$defs/pair" + } + }, + "claimBoundary": { + "type": "string", + "enum": [ + "calibration-only", + "no-support", + "directional-signal", + "repeatable-workflow-value-support-candidate", + "public-benchmark-support" + ] + }, + "bannedClaims": { + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "minLength": 1 + } + } + }, + "$defs": { + "lane": { + "type": "object", + "additionalProperties": false, + "required": [ + "laneId", + "label", + "processType", + "enabled", + "allowedTools", + "forbiddenAdvantages", + "requiredEvidence" + ], + "properties": { + "laneId": { + "type": "string", + "pattern": "^[A-Z][A-Z0-9_-]{0,15}$" + }, + "label": { + "type": "string", + "minLength": 1 + }, + "processType": { + "type": "string", + "enum": [ + "naked-model", + "workflow-system", + "scripted-agent", + "human-operated", + "other" + ] + }, + "enabled": { + "type": "boolean" + }, + "allowedTools": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + } + }, + "forbiddenAdvantages": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + } + }, + "requiredEvidence": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + } + } + } + }, + "pair": { + "type": "object", + "additionalProperties": false, + "required": [ + "pairId", + "taskSeed", + "scenarioRef", + "enabledLanes", + "notes" + ], + "properties": { + "pairId": { + "type": "string", + "pattern": "^[a-z0-9][a-z0-9-]{2,100}$" + }, + "taskSeed": { + "type": "integer" + }, + "scenarioRef": { + "type": "string", + "minLength": 1 + }, + "enabledLanes": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "type": "string", + "pattern": "^[A-Z][A-Z0-9_-]{0,15}$" + } + }, + "notes": { + "type": "string" + } + } + } + } +} diff --git a/v3/examples/invalid/campaign-boolean-seed.json b/v3/examples/invalid/campaign-boolean-seed.json new file mode 100644 index 0000000..0c2793f --- /dev/null +++ b/v3/examples/invalid/campaign-boolean-seed.json @@ -0,0 +1,107 @@ +{ + "schemaVersion": "2000m.v3.campaign.v1", + "campaignId": "v3-foundation-fixture", + "title": "v3 foundation schema fixture", + "status": "draft", + "protocolFreeze": { + "freezeBeforeLiveResults": true, + "noProtocolMutationAfterLiveResults": true, + "mutationAfterLiveResultsPolicy": "affected-runs-calibration-only", + "benchmarkCommit": "0000000-foundation-fixture", + "driverProtocolRef": "v3/2000m.driver.v3.md", + "mechanicalAcSpecRef": "v3/MECHANICAL_AC_SPEC.md", + "visualRubricRef": "v3/VISUAL_RUBRIC.md", + "scoringModelRef": "v3/SCORING_MODEL.md", + "sandboxPolicyRef": "v3/SANDBOX_AND_RESOURCE_POLICY.md", + "scenarioRefs": [ + "v3/examples/valid/run-record.foundation.json" + ], + "feedbackPacketRef": "v3/WORKFLOW_RESILIENCE_PROTOCOL.md#phase-3--reviewer-injection", + "decisionRulesRef": "v3/EVIDENCE_AND_CLAIMS.md#claim-ladder", + "captureWindows": [ + "early-game", + "monster-pressure" + ], + "visualSeeds": [ + 1101, + 2202 + ], + "hiddenChallengePolicy": "declared-public-hidden-split" + }, + "controls": { + "model": "fixture-model", + "runtime": "fixture-runtime", + "generationCap": 2, + "promptBudget": 12000, + "scorerFeedbackBudget": 1, + "reviewerFeedbackBudget": 1, + "contextWipePhase": "after-first-scored-generation", + "samePromptBudget": true, + "sameFeedbackPackets": true, + "sameScorerDiagnostics": true, + "sameVisualSeeds": true, + "sameReviewerBudget": true, + "wallClockPolicy": "diagnostic-only", + "networkPolicy": "disabled-during-scoring" + }, + "lanes": [ + { + "laneId": "A", + "label": "unstructured baseline", + "processType": "naked-model", + "enabled": true, + "allowedTools": [ + "public spec", + "scorer feedback" + ], + "forbiddenAdvantages": [ + "lane-specific hidden hints", + "post-hoc scorer edits" + ], + "requiredEvidence": [ + "generic run record", + "mechanical result", + "visual status" + ] + }, + { + "laneId": "B", + "label": "structured workflow lane", + "processType": "workflow-system", + "enabled": true, + "allowedTools": [ + "public spec", + "same scorer feedback" + ], + "forbiddenAdvantages": [ + "lane-specific hidden hints", + "post-hoc scorer edits" + ], + "requiredEvidence": [ + "generic run record", + "mechanical result", + "visual status" + ] + } + ], + "pairs": [ + { + "pairId": "fixture-pair-101", + "taskSeed": true, + "scenarioRef": "v3/examples/valid/run-record.foundation.json", + "enabledLanes": [ + "A", + "B" + ], + "notes": "Schema fixture only; not a pilot." + } + ], + "claimBoundary": "calibration-only", + "bannedClaims": [ + "makes the model smarter", + "proves workflow superiority", + "adoption proof", + "public contender result from a fixture", + "high mechanical score means a good game" + ] +} diff --git a/v3/examples/invalid/campaign-framework-specific-required-field.json b/v3/examples/invalid/campaign-framework-specific-required-field.json new file mode 100644 index 0000000..2d72273 --- /dev/null +++ b/v3/examples/invalid/campaign-framework-specific-required-field.json @@ -0,0 +1,108 @@ +{ + "schemaVersion": "2000m.v3.campaign.v1", + "campaignId": "v3-foundation-fixture", + "title": "v3 foundation schema fixture", + "status": "draft", + "protocolFreeze": { + "freezeBeforeLiveResults": true, + "noProtocolMutationAfterLiveResults": true, + "mutationAfterLiveResultsPolicy": "affected-runs-calibration-only", + "benchmarkCommit": "0000000-foundation-fixture", + "driverProtocolRef": "v3/2000m.driver.v3.md", + "mechanicalAcSpecRef": "v3/MECHANICAL_AC_SPEC.md", + "visualRubricRef": "v3/VISUAL_RUBRIC.md", + "scoringModelRef": "v3/SCORING_MODEL.md", + "sandboxPolicyRef": "v3/SANDBOX_AND_RESOURCE_POLICY.md", + "scenarioRefs": [ + "v3/examples/valid/run-record.foundation.json" + ], + "feedbackPacketRef": "v3/WORKFLOW_RESILIENCE_PROTOCOL.md#phase-3--reviewer-injection", + "decisionRulesRef": "v3/EVIDENCE_AND_CLAIMS.md#claim-ladder", + "captureWindows": [ + "early-game", + "monster-pressure" + ], + "visualSeeds": [ + 1101, + 2202 + ], + "hiddenChallengePolicy": "declared-public-hidden-split" + }, + "controls": { + "model": "fixture-model", + "runtime": "fixture-runtime", + "generationCap": 2, + "promptBudget": 12000, + "scorerFeedbackBudget": 1, + "reviewerFeedbackBudget": 1, + "contextWipePhase": "after-first-scored-generation", + "samePromptBudget": true, + "sameFeedbackPackets": true, + "sameScorerDiagnostics": true, + "sameVisualSeeds": true, + "sameReviewerBudget": true, + "wallClockPolicy": "diagnostic-only", + "networkPolicy": "disabled-during-scoring" + }, + "lanes": [ + { + "laneId": "A", + "label": "unstructured baseline", + "processType": "naked-model", + "enabled": true, + "allowedTools": [ + "public spec", + "scorer feedback" + ], + "forbiddenAdvantages": [ + "lane-specific hidden hints", + "post-hoc scorer edits" + ], + "requiredEvidence": [ + "generic run record", + "mechanical result", + "visual status" + ] + }, + { + "laneId": "B", + "label": "structured workflow lane", + "processType": "workflow-system", + "enabled": true, + "allowedTools": [ + "public spec", + "same scorer feedback" + ], + "forbiddenAdvantages": [ + "lane-specific hidden hints", + "post-hoc scorer edits" + ], + "requiredEvidence": [ + "generic run record", + "mechanical result", + "visual status" + ], + "frameworkRequiredField": "specific-workflow-framework-plan-ref" + } + ], + "pairs": [ + { + "pairId": "fixture-pair-101", + "taskSeed": 101, + "scenarioRef": "v3/examples/valid/run-record.foundation.json", + "enabledLanes": [ + "A", + "B" + ], + "notes": "Schema fixture only; not a pilot." + } + ], + "claimBoundary": "calibration-only", + "bannedClaims": [ + "makes the model smarter", + "proves workflow superiority", + "adoption proof", + "public contender result from a fixture", + "high mechanical score means a good game" + ] +} diff --git a/v3/examples/invalid/manifest-private-path.json b/v3/examples/invalid/manifest-private-path.json new file mode 100644 index 0000000..f53380b --- /dev/null +++ b/v3/examples/invalid/manifest-private-path.json @@ -0,0 +1,61 @@ +{ + "schemaVersion": "2000m.v3.manifest.v1", + "protocolVersion": "2000m.driver.v3", + "language": "rust", + "driver": { + "command": "cargo", + "args": [ + "run", + "--quiet", + "--bin", + "driver" + ], + "timeoutSeconds": 30 + }, + "capture": { + "command": "cargo", + "args": [ + "run", + "--quiet", + "--bin", + "capture", + "--", + "--seed", + "{seed}", + "--window", + "{window}", + "--out", + "{out}" + ], + "outputs": { + "screenshot": "{out}/screenshot.png", + "replay": "{out}/replay.gif", + "frames": "{out}/frames.json", + "rubricMetadata": "{out}/rubric-metadata.json" + }, + "metadata": { + "placeholders": [ + "{seed}", + "{window}", + "{out}" + ], + "deterministicForSameSeedWindowAndArtifact": true + } + }, + "playable": { + "command": "cargo", + "args": [ + "run", + "--quiet", + "--bin", + "game" + ], + "urlOrPath": "file://local-only/private-build/index.html" + }, + "assets": { + "license": "original-homage-assets-only", + "attestation": "No copied or extracted SkiFree assets are included.", + "sourceRefs": [] + }, + "notes": "Foundation fixture; not a live contender." +} diff --git a/v3/examples/invalid/result-mutated-frozen-protocol.json b/v3/examples/invalid/result-mutated-frozen-protocol.json new file mode 100644 index 0000000..6017587 --- /dev/null +++ b/v3/examples/invalid/result-mutated-frozen-protocol.json @@ -0,0 +1,64 @@ +{ + "schemaVersion": "2000m.v3.result.v1", + "campaignId": "v3-foundation-fixture", + "scenarioId": "v3-foundation-scenario", + "taskSeed": 101, + "laneId": "A", + "entrant": { + "model": "fixture-model", + "runtime": "fixture-runtime", + "processType": "naked-model" + }, + "protocolFreeze": { + "changedAfterLiveResults": true, + "scorerMutationObserved": true, + "calibrationOnlyIfChanged": true + }, + "mechanical": { + "ranked": false, + "protocolVersion": "2000m.driver.v3", + "passCount": 0, + "totalAcs": 24, + "compositeScore": 0, + "determinism": { + "pass": true, + "details": "fixture only" + }, + "failedAcs": [ + "M01" + ], + "hiddenChallengeSummary": "not run", + "regressionSummary": "not run", + "resultJsonRef": "fixtures/mechanical-result.json" + }, + "visual": { + "ranked": false, + "blockReason": "missing-native-capture-or-playable-surface", + "visualPackageRef": "", + "captureDeterminism": "blocked", + "rubricRecordRef": "" + }, + "workflow": { + "contextWipeRecoveryScore": 0, + "feedbackDecisionScore": 0, + "regressionProtectionScore": 0, + "impossibleRequirementHandlingScore": 0, + "handoffScore": 0, + "finalRecommendation": "redesign", + "rationaleRefs": [ + "v3/SPEC_GAPS_AND_DECISIONS.md" + ] + }, + "evidence": { + "replayable": false, + "publicSafe": true, + "privateRefsBlocked": false, + "compactSummaryRef": "fixtures/summary.md", + "requiredRefsMissing": [], + "claimBoundary": "calibration-only" + }, + "claimBoundary": "calibration-only", + "warnings": [ + "Foundation fixture; not a contender result." + ] +} diff --git a/v3/examples/invalid/result-ranked-visual-missing-evidence.json b/v3/examples/invalid/result-ranked-visual-missing-evidence.json new file mode 100644 index 0000000..195ae62 --- /dev/null +++ b/v3/examples/invalid/result-ranked-visual-missing-evidence.json @@ -0,0 +1,64 @@ +{ + "schemaVersion": "2000m.v3.result.v1", + "campaignId": "v3-foundation-fixture", + "scenarioId": "v3-foundation-scenario", + "taskSeed": 101, + "laneId": "A", + "entrant": { + "model": "fixture-model", + "runtime": "fixture-runtime", + "processType": "naked-model" + }, + "protocolFreeze": { + "changedAfterLiveResults": false, + "scorerMutationObserved": false, + "calibrationOnlyIfChanged": true + }, + "mechanical": { + "ranked": false, + "protocolVersion": "2000m.driver.v3", + "passCount": 0, + "totalAcs": 24, + "compositeScore": 0, + "determinism": { + "pass": true, + "details": "fixture only" + }, + "failedAcs": [ + "M01" + ], + "hiddenChallengeSummary": "not run", + "regressionSummary": "not run", + "resultJsonRef": "fixtures/mechanical-result.json" + }, + "visual": { + "ranked": true, + "blockReason": "none", + "visualPackageRef": "", + "captureDeterminism": "blocked", + "rubricRecordRef": "" + }, + "workflow": { + "contextWipeRecoveryScore": 0, + "feedbackDecisionScore": 0, + "regressionProtectionScore": 0, + "impossibleRequirementHandlingScore": 0, + "handoffScore": 0, + "finalRecommendation": "redesign", + "rationaleRefs": [ + "v3/SPEC_GAPS_AND_DECISIONS.md" + ] + }, + "evidence": { + "replayable": false, + "publicSafe": true, + "privateRefsBlocked": false, + "compactSummaryRef": "fixtures/summary.md", + "requiredRefsMissing": [], + "claimBoundary": "calibration-only" + }, + "claimBoundary": "calibration-only", + "warnings": [ + "Foundation fixture; not a contender result." + ] +} diff --git a/v3/examples/invalid/result-unsupported-claim.json b/v3/examples/invalid/result-unsupported-claim.json new file mode 100644 index 0000000..ac51695 --- /dev/null +++ b/v3/examples/invalid/result-unsupported-claim.json @@ -0,0 +1,65 @@ +{ + "schemaVersion": "2000m.v3.result.v1", + "campaignId": "v3-foundation-fixture", + "scenarioId": "v3-foundation-scenario", + "taskSeed": 101, + "laneId": "A", + "entrant": { + "model": "fixture-model", + "runtime": "fixture-runtime", + "processType": "naked-model" + }, + "protocolFreeze": { + "changedAfterLiveResults": false, + "scorerMutationObserved": false, + "calibrationOnlyIfChanged": true + }, + "mechanical": { + "ranked": false, + "protocolVersion": "2000m.driver.v3", + "passCount": 0, + "totalAcs": 24, + "compositeScore": 0, + "determinism": { + "pass": true, + "details": "fixture only" + }, + "failedAcs": [ + "M01" + ], + "hiddenChallengeSummary": "not run", + "regressionSummary": "not run", + "resultJsonRef": "fixtures/mechanical-result.json" + }, + "visual": { + "ranked": false, + "blockReason": "missing-native-capture-or-playable-surface", + "visualPackageRef": "", + "captureDeterminism": "blocked", + "rubricRecordRef": "" + }, + "workflow": { + "contextWipeRecoveryScore": 0, + "feedbackDecisionScore": 0, + "regressionProtectionScore": 0, + "impossibleRequirementHandlingScore": 0, + "handoffScore": 0, + "finalRecommendation": "redesign", + "rationaleRefs": [ + "v3/SPEC_GAPS_AND_DECISIONS.md" + ] + }, + "evidence": { + "replayable": false, + "publicSafe": true, + "privateRefsBlocked": false, + "compactSummaryRef": "fixtures/summary.md", + "requiredRefsMissing": [], + "claimBoundary": "calibration-only" + }, + "claimBoundary": "public-benchmark-support", + "warnings": [ + "Foundation fixture; not a contender result.", + "This proves workflow superiority and adoption proof from one fixture." + ] +} diff --git a/v3/examples/invalid/run-record-invalid-enum.json b/v3/examples/invalid/run-record-invalid-enum.json new file mode 100644 index 0000000..ebc2721 --- /dev/null +++ b/v3/examples/invalid/run-record-invalid-enum.json @@ -0,0 +1,95 @@ +{ + "schemaVersion": "2000m.v3.run-record.v1", + "campaignId": "v3-foundation-fixture", + "scenarioId": "v3-foundation-scenario", + "taskSeed": 101, + "laneId": "A", + "entrant": { + "model": "fixture-model", + "runtime": "fixture-runtime", + "processType": "naked-model" + }, + "artifact": { + "repoOrPath": "fixtures/produced-artifact", + "commitOrDigest": "sha256:0000000000000000000000000000000000000000000000000000000000000000", + "manifestRef": "v3/examples/valid/manifest.full.json", + "changedFilesRef": "fixtures/changed-files.txt" + }, + "protocolFreeze": { + "frozen": true, + "frozenCampaignRef": "v3/examples/valid/campaign.foundation.json", + "changedAfterLiveResults": false, + "scorerMutationObserved": false + }, + "phases": [ + { + "phaseId": "phase-1", + "kind": "controller-autonomy", + "outputs": { + "buildCommand": "cargo build", + "scoreCommand": "cargo run -p m2000-v3-conformance -- fixtures/produced-artifact" + }, + "feedbackDecisions": [ + { + "feedbackId": "review-1", + "decision": "deferred", + "rationale": "Fixture has no live reviewer packet.", + "evidenceRef": "v3/WORKFLOW_RESILIENCE_PROTOCOL.md" + } + ], + "evidenceRefs": [ + { + "label": "prompt", + "ref": "fixtures/prompt.md", + "kind": "prompt" + }, + { + "label": "score", + "ref": "fixtures/mechanical-result.json", + "kind": "conformance-json" + } + ] + } + ], + "mechanical": { + "ranked": false, + "protocolVersion": "2000m.driver.v3", + "resultJsonRef": "fixtures/mechanical-result.json", + "passCount": 0, + "totalAcs": 24, + "failedAcs": [ + "M01" + ] + }, + "visual": { + "ranked": false, + "blockReason": "missing-native-capture-or-playable-surface", + "visualPackageRef": "", + "captureCommandResultRef": "fixtures/capture.log" + }, + "workflow": { + "contextWipeStatus": "not-reached", + "feedbackDecisionStatus": "not-reached", + "regressionStatus": "not-reached", + "handoffStatus": "not-reached" + }, + "evidence": { + "publicSafe": true, + "privateRefsBlocked": false, + "requiredRefsMissing": [], + "claimBoundary": "calibration-only", + "refs": [ + { + "label": "summary", + "ref": "fixtures/summary.md", + "kind": "summary" + } + ] + }, + "finalRecommendation": { + "decision": "redesign", + "rationaleRefs": [ + "v3/SPEC_GAPS_AND_DECISIONS.md" + ] + } +} diff --git a/v3/examples/invalid/run-record-ranked-visual-missing-evidence.json b/v3/examples/invalid/run-record-ranked-visual-missing-evidence.json new file mode 100644 index 0000000..2aa3f08 --- /dev/null +++ b/v3/examples/invalid/run-record-ranked-visual-missing-evidence.json @@ -0,0 +1,95 @@ +{ + "schemaVersion": "2000m.v3.run-record.v1", + "campaignId": "v3-foundation-fixture", + "scenarioId": "v3-foundation-scenario", + "taskSeed": 101, + "laneId": "A", + "entrant": { + "model": "fixture-model", + "runtime": "fixture-runtime", + "processType": "naked-model" + }, + "artifact": { + "repoOrPath": "fixtures/produced-artifact", + "commitOrDigest": "sha256:0000000000000000000000000000000000000000000000000000000000000000", + "manifestRef": "v3/examples/valid/manifest.full.json", + "changedFilesRef": "fixtures/changed-files.txt" + }, + "protocolFreeze": { + "frozen": true, + "frozenCampaignRef": "v3/examples/valid/campaign.foundation.json", + "changedAfterLiveResults": false, + "scorerMutationObserved": false + }, + "phases": [ + { + "phaseId": "phase-1", + "kind": "initial-attempt", + "outputs": { + "buildCommand": "cargo build", + "scoreCommand": "cargo run -p m2000-v3-conformance -- fixtures/produced-artifact" + }, + "feedbackDecisions": [ + { + "feedbackId": "review-1", + "decision": "deferred", + "rationale": "Fixture has no live reviewer packet.", + "evidenceRef": "v3/WORKFLOW_RESILIENCE_PROTOCOL.md" + } + ], + "evidenceRefs": [ + { + "label": "prompt", + "ref": "fixtures/prompt.md", + "kind": "prompt" + }, + { + "label": "score", + "ref": "fixtures/mechanical-result.json", + "kind": "conformance-json" + } + ] + } + ], + "mechanical": { + "ranked": false, + "protocolVersion": "2000m.driver.v3", + "resultJsonRef": "fixtures/mechanical-result.json", + "passCount": 0, + "totalAcs": 24, + "failedAcs": [ + "M01" + ] + }, + "visual": { + "ranked": true, + "blockReason": "none", + "visualPackageRef": "", + "captureCommandResultRef": "" + }, + "workflow": { + "contextWipeStatus": "not-reached", + "feedbackDecisionStatus": "not-reached", + "regressionStatus": "not-reached", + "handoffStatus": "not-reached" + }, + "evidence": { + "publicSafe": true, + "privateRefsBlocked": false, + "requiredRefsMissing": [], + "claimBoundary": "calibration-only", + "refs": [ + { + "label": "summary", + "ref": "fixtures/summary.md", + "kind": "summary" + } + ] + }, + "finalRecommendation": { + "decision": "redesign", + "rationaleRefs": [ + "v3/SPEC_GAPS_AND_DECISIONS.md" + ] + } +} diff --git a/v3/examples/invalid/visual-package-missing-capture-metadata.json b/v3/examples/invalid/visual-package-missing-capture-metadata.json new file mode 100644 index 0000000..7a7b011 --- /dev/null +++ b/v3/examples/invalid/visual-package-missing-capture-metadata.json @@ -0,0 +1,39 @@ +{ + "schemaVersion": "2000m.v3.visual-package.v1", + "packageId": "v3-foundation-visual-package", + "campaignId": "v3-foundation-fixture", + "scenarioId": "v3-foundation-scenario", + "taskSeed": 101, + "anonymized": true, + "mappingSealedBeforeReview": true, + "artifactDigest": "sha256:0000000000000000000000000000000000000000000000000000000000000000", + "windows": [ + { + "window": "early-game", + "seed": 1101, + "captureCommand": "cargo run --quiet --bin capture -- --seed 1101 --window early-game --out visual-package/A/early-game", + "screenshotRef": "visual-package/screenshots/A-early-game.png", + "replayRef": "visual-package/gifs/A-early-game.gif", + "rubricMetadataRef": "visual-package/frames/A-early-game.rubric-metadata.json", + "fps": 30, + "frameCount": 120, + "inputSequenceRef": "visual-package/inputs/early-game.json", + "stateChecksum": "sha256:0000000000000000000000000000000000000000000000000000000000000000", + "frameChecksum": "sha256:1111111111111111111111111111111111111111111111111111111111111111", + "warnings": [] + } + ], + "artifacts": { + "blindContactSheetRef": "visual-package/blind-contact-sheet.png", + "blindLabelMapRef": "visual-package/blind-label-map.json", + "screenshotsDirRef": "visual-package/screenshots", + "replaysDirRef": "visual-package/gifs", + "framesDirRef": "visual-package/frames" + }, + "rubric": { + "rubricRef": "v3/VISUAL_RUBRIC.md", + "rubricRecordRef": "visual-package/rubric-record.json", + "reviewerCount": 1, + "ownerTasteSeparated": true + } +} diff --git a/v3/examples/valid/campaign.foundation.json b/v3/examples/valid/campaign.foundation.json new file mode 100644 index 0000000..f872bcf --- /dev/null +++ b/v3/examples/valid/campaign.foundation.json @@ -0,0 +1,107 @@ +{ + "schemaVersion": "2000m.v3.campaign.v1", + "campaignId": "v3-foundation-fixture", + "title": "v3 foundation schema fixture", + "status": "draft", + "protocolFreeze": { + "freezeBeforeLiveResults": true, + "noProtocolMutationAfterLiveResults": true, + "mutationAfterLiveResultsPolicy": "affected-runs-calibration-only", + "benchmarkCommit": "0000000-foundation-fixture", + "driverProtocolRef": "v3/2000m.driver.v3.md", + "mechanicalAcSpecRef": "v3/MECHANICAL_AC_SPEC.md", + "visualRubricRef": "v3/VISUAL_RUBRIC.md", + "scoringModelRef": "v3/SCORING_MODEL.md", + "sandboxPolicyRef": "v3/SANDBOX_AND_RESOURCE_POLICY.md", + "scenarioRefs": [ + "v3/examples/valid/run-record.foundation.json" + ], + "feedbackPacketRef": "v3/WORKFLOW_RESILIENCE_PROTOCOL.md#phase-3--reviewer-injection", + "decisionRulesRef": "v3/EVIDENCE_AND_CLAIMS.md#claim-ladder", + "captureWindows": [ + "early-game", + "monster-pressure" + ], + "visualSeeds": [ + 1101, + 2202 + ], + "hiddenChallengePolicy": "declared-public-hidden-split" + }, + "controls": { + "model": "fixture-model", + "runtime": "fixture-runtime", + "generationCap": 2, + "promptBudget": 12000, + "scorerFeedbackBudget": 1, + "reviewerFeedbackBudget": 1, + "contextWipePhase": "after-first-scored-generation", + "samePromptBudget": true, + "sameFeedbackPackets": true, + "sameScorerDiagnostics": true, + "sameVisualSeeds": true, + "sameReviewerBudget": true, + "wallClockPolicy": "diagnostic-only", + "networkPolicy": "disabled-during-scoring" + }, + "lanes": [ + { + "laneId": "A", + "label": "unstructured baseline", + "processType": "naked-model", + "enabled": true, + "allowedTools": [ + "public spec", + "scorer feedback" + ], + "forbiddenAdvantages": [ + "lane-specific hidden hints", + "post-hoc scorer edits" + ], + "requiredEvidence": [ + "generic run record", + "mechanical result", + "visual status" + ] + }, + { + "laneId": "B", + "label": "structured workflow lane", + "processType": "workflow-system", + "enabled": true, + "allowedTools": [ + "public spec", + "same scorer feedback" + ], + "forbiddenAdvantages": [ + "lane-specific hidden hints", + "post-hoc scorer edits" + ], + "requiredEvidence": [ + "generic run record", + "mechanical result", + "visual status" + ] + } + ], + "pairs": [ + { + "pairId": "fixture-pair-101", + "taskSeed": 101, + "scenarioRef": "v3/examples/valid/run-record.foundation.json", + "enabledLanes": [ + "A", + "B" + ], + "notes": "Schema fixture only; not a pilot." + } + ], + "claimBoundary": "calibration-only", + "bannedClaims": [ + "makes the model smarter", + "proves workflow superiority", + "adoption proof", + "public contender result from a fixture", + "high mechanical score means a good game" + ] +} diff --git a/v3/examples/valid/manifest.full.json b/v3/examples/valid/manifest.full.json new file mode 100644 index 0000000..5599446 --- /dev/null +++ b/v3/examples/valid/manifest.full.json @@ -0,0 +1,61 @@ +{ + "schemaVersion": "2000m.v3.manifest.v1", + "protocolVersion": "2000m.driver.v3", + "language": "rust", + "driver": { + "command": "cargo", + "args": [ + "run", + "--quiet", + "--bin", + "driver" + ], + "timeoutSeconds": 30 + }, + "capture": { + "command": "cargo", + "args": [ + "run", + "--quiet", + "--bin", + "capture", + "--", + "--seed", + "{seed}", + "--window", + "{window}", + "--out", + "{out}" + ], + "outputs": { + "screenshot": "{out}/screenshot.png", + "replay": "{out}/replay.gif", + "frames": "{out}/frames.json", + "rubricMetadata": "{out}/rubric-metadata.json" + }, + "metadata": { + "placeholders": [ + "{seed}", + "{window}", + "{out}" + ], + "deterministicForSameSeedWindowAndArtifact": true + } + }, + "playable": { + "command": "cargo", + "args": [ + "run", + "--quiet", + "--bin", + "game" + ], + "urlOrPath": "target/2000m/index.html" + }, + "assets": { + "license": "original-homage-assets-only", + "attestation": "No copied or extracted SkiFree assets are included.", + "sourceRefs": [] + }, + "notes": "Foundation fixture; not a live contender." +} diff --git a/v3/examples/valid/result.foundation.json b/v3/examples/valid/result.foundation.json new file mode 100644 index 0000000..f289cf2 --- /dev/null +++ b/v3/examples/valid/result.foundation.json @@ -0,0 +1,64 @@ +{ + "schemaVersion": "2000m.v3.result.v1", + "campaignId": "v3-foundation-fixture", + "scenarioId": "v3-foundation-scenario", + "taskSeed": 101, + "laneId": "A", + "entrant": { + "model": "fixture-model", + "runtime": "fixture-runtime", + "processType": "naked-model" + }, + "protocolFreeze": { + "changedAfterLiveResults": false, + "scorerMutationObserved": false, + "calibrationOnlyIfChanged": true + }, + "mechanical": { + "ranked": false, + "protocolVersion": "2000m.driver.v3", + "passCount": 0, + "totalAcs": 24, + "compositeScore": 0, + "determinism": { + "pass": true, + "details": "fixture only" + }, + "failedAcs": [ + "M01" + ], + "hiddenChallengeSummary": "not run", + "regressionSummary": "not run", + "resultJsonRef": "fixtures/mechanical-result.json" + }, + "visual": { + "ranked": false, + "blockReason": "missing-native-capture-or-playable-surface", + "visualPackageRef": "", + "captureDeterminism": "blocked", + "rubricRecordRef": "" + }, + "workflow": { + "contextWipeRecoveryScore": 0, + "feedbackDecisionScore": 0, + "regressionProtectionScore": 0, + "impossibleRequirementHandlingScore": 0, + "handoffScore": 0, + "finalRecommendation": "redesign", + "rationaleRefs": [ + "v3/SPEC_GAPS_AND_DECISIONS.md" + ] + }, + "evidence": { + "replayable": false, + "publicSafe": true, + "privateRefsBlocked": false, + "compactSummaryRef": "fixtures/summary.md", + "requiredRefsMissing": [], + "claimBoundary": "calibration-only" + }, + "claimBoundary": "calibration-only", + "warnings": [ + "Foundation fixture; not a contender result." + ] +} diff --git a/v3/examples/valid/run-record.foundation.json b/v3/examples/valid/run-record.foundation.json new file mode 100644 index 0000000..4a48083 --- /dev/null +++ b/v3/examples/valid/run-record.foundation.json @@ -0,0 +1,95 @@ +{ + "schemaVersion": "2000m.v3.run-record.v1", + "campaignId": "v3-foundation-fixture", + "scenarioId": "v3-foundation-scenario", + "taskSeed": 101, + "laneId": "A", + "entrant": { + "model": "fixture-model", + "runtime": "fixture-runtime", + "processType": "naked-model" + }, + "artifact": { + "repoOrPath": "fixtures/produced-artifact", + "commitOrDigest": "sha256:0000000000000000000000000000000000000000000000000000000000000000", + "manifestRef": "v3/examples/valid/manifest.full.json", + "changedFilesRef": "fixtures/changed-files.txt" + }, + "protocolFreeze": { + "frozen": true, + "frozenCampaignRef": "v3/examples/valid/campaign.foundation.json", + "changedAfterLiveResults": false, + "scorerMutationObserved": false + }, + "phases": [ + { + "phaseId": "phase-1", + "kind": "initial-attempt", + "outputs": { + "buildCommand": "cargo build", + "scoreCommand": "cargo run -p m2000-v3-conformance -- fixtures/produced-artifact" + }, + "feedbackDecisions": [ + { + "feedbackId": "review-1", + "decision": "deferred", + "rationale": "Fixture has no live reviewer packet.", + "evidenceRef": "v3/WORKFLOW_RESILIENCE_PROTOCOL.md" + } + ], + "evidenceRefs": [ + { + "label": "prompt", + "ref": "fixtures/prompt.md", + "kind": "prompt" + }, + { + "label": "score", + "ref": "fixtures/mechanical-result.json", + "kind": "conformance-json" + } + ] + } + ], + "mechanical": { + "ranked": false, + "protocolVersion": "2000m.driver.v3", + "resultJsonRef": "fixtures/mechanical-result.json", + "passCount": 0, + "totalAcs": 24, + "failedAcs": [ + "M01" + ] + }, + "visual": { + "ranked": false, + "blockReason": "missing-native-capture-or-playable-surface", + "visualPackageRef": "", + "captureCommandResultRef": "fixtures/capture.log" + }, + "workflow": { + "contextWipeStatus": "not-reached", + "feedbackDecisionStatus": "not-reached", + "regressionStatus": "not-reached", + "handoffStatus": "not-reached" + }, + "evidence": { + "publicSafe": true, + "privateRefsBlocked": false, + "requiredRefsMissing": [], + "claimBoundary": "calibration-only", + "refs": [ + { + "label": "summary", + "ref": "fixtures/summary.md", + "kind": "summary" + } + ] + }, + "finalRecommendation": { + "decision": "redesign", + "rationaleRefs": [ + "v3/SPEC_GAPS_AND_DECISIONS.md" + ] + } +} diff --git a/v3/examples/valid/visual-package.foundation.json b/v3/examples/valid/visual-package.foundation.json new file mode 100644 index 0000000..2285555 --- /dev/null +++ b/v3/examples/valid/visual-package.foundation.json @@ -0,0 +1,40 @@ +{ + "schemaVersion": "2000m.v3.visual-package.v1", + "packageId": "v3-foundation-visual-package", + "campaignId": "v3-foundation-fixture", + "scenarioId": "v3-foundation-scenario", + "taskSeed": 101, + "anonymized": true, + "mappingSealedBeforeReview": true, + "artifactDigest": "sha256:0000000000000000000000000000000000000000000000000000000000000000", + "windows": [ + { + "window": "early-game", + "seed": 1101, + "captureCommand": "cargo run --quiet --bin capture -- --seed 1101 --window early-game --out visual-package/A/early-game", + "screenshotRef": "visual-package/screenshots/A-early-game.png", + "replayRef": "visual-package/gifs/A-early-game.gif", + "frameMetadataRef": "visual-package/frames/A-early-game.frames.json", + "rubricMetadataRef": "visual-package/frames/A-early-game.rubric-metadata.json", + "fps": 30, + "frameCount": 120, + "inputSequenceRef": "visual-package/inputs/early-game.json", + "stateChecksum": "sha256:0000000000000000000000000000000000000000000000000000000000000000", + "frameChecksum": "sha256:1111111111111111111111111111111111111111111111111111111111111111", + "warnings": [] + } + ], + "artifacts": { + "blindContactSheetRef": "visual-package/blind-contact-sheet.png", + "blindLabelMapRef": "visual-package/blind-label-map.json", + "screenshotsDirRef": "visual-package/screenshots", + "replaysDirRef": "visual-package/gifs", + "framesDirRef": "visual-package/frames" + }, + "rubric": { + "rubricRef": "v3/VISUAL_RUBRIC.md", + "rubricRecordRef": "visual-package/rubric-record.json", + "reviewerCount": 1, + "ownerTasteSeparated": true + } +} diff --git a/v3/manifest.schema.json b/v3/manifest.schema.json new file mode 100644 index 0000000..a30dbc1 --- /dev/null +++ b/v3/manifest.schema.json @@ -0,0 +1,186 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/graphanov/2000m/v3/manifest.schema.json", + "title": "2000m v3 produced-artifact manifest", + "type": "object", + "additionalProperties": false, + "required": [ + "schemaVersion", + "protocolVersion", + "language", + "driver", + "assets" + ], + "properties": { + "schemaVersion": { + "const": "2000m.v3.manifest.v1" + }, + "protocolVersion": { + "const": "2000m.driver.v3" + }, + "language": { + "type": "string", + "enum": [ + "rust" + ] + }, + "driver": { + "type": "object", + "additionalProperties": false, + "required": [ + "command", + "args" + ], + "properties": { + "command": { + "type": "string", + "minLength": 1 + }, + "args": { + "type": "array", + "items": { + "type": "string" + } + }, + "cwd": { + "type": "string", + "description": "Optional repo-relative working directory. Public manifests must not use private/local absolute paths." + }, + "timeoutSeconds": { + "type": "integer", + "minimum": 1 + } + } + }, + "capture": { + "type": "object", + "additionalProperties": false, + "required": [ + "command", + "args", + "outputs", + "metadata" + ], + "properties": { + "command": { + "type": "string", + "minLength": 1 + }, + "args": { + "type": "array", + "items": { + "type": "string" + } + }, + "outputs": { + "type": "object", + "additionalProperties": false, + "required": [ + "screenshot", + "replay", + "frames", + "rubricMetadata" + ], + "properties": { + "screenshot": { + "type": "string", + "minLength": 1 + }, + "replay": { + "type": "string", + "minLength": 1 + }, + "frames": { + "type": "string", + "minLength": 1 + }, + "rubricMetadata": { + "type": "string", + "minLength": 1 + } + } + }, + "metadata": { + "type": "object", + "additionalProperties": false, + "required": [ + "placeholders", + "deterministicForSameSeedWindowAndArtifact" + ], + "properties": { + "placeholders": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "{seed}", + "{window}", + "{out}" + ] + } + }, + "deterministicForSameSeedWindowAndArtifact": { + "type": "boolean" + } + } + } + } + }, + "playable": { + "type": "object", + "additionalProperties": false, + "required": [ + "command", + "args", + "urlOrPath" + ], + "properties": { + "command": { + "type": "string", + "minLength": 1 + }, + "args": { + "type": "array", + "items": { + "type": "string" + } + }, + "urlOrPath": { + "type": "string", + "minLength": 1 + } + } + }, + "assets": { + "type": "object", + "additionalProperties": false, + "required": [ + "license", + "attestation" + ], + "properties": { + "license": { + "type": "string", + "enum": [ + "original-homage-assets-only", + "no-assets", + "other-publicly-licensed-assets" + ] + }, + "attestation": { + "type": "string", + "minLength": 1 + }, + "sourceRefs": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "notes": { + "type": "string" + } + } +} diff --git a/v3/result.schema.json b/v3/result.schema.json new file mode 100644 index 0000000..5f3c2c9 --- /dev/null +++ b/v3/result.schema.json @@ -0,0 +1,408 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/graphanov/2000m/v3/result.schema.json", + "title": "2000m v3 separated-track result", + "type": "object", + "additionalProperties": false, + "required": [ + "schemaVersion", + "campaignId", + "scenarioId", + "taskSeed", + "laneId", + "entrant", + "protocolFreeze", + "mechanical", + "visual", + "workflow", + "evidence", + "claimBoundary", + "warnings" + ], + "properties": { + "schemaVersion": { + "const": "2000m.v3.result.v1" + }, + "campaignId": { + "type": "string", + "pattern": "^[a-z0-9][a-z0-9-]{2,100}$" + }, + "scenarioId": { + "type": "string", + "pattern": "^[a-z0-9][a-z0-9-]{2,100}$" + }, + "taskSeed": { + "type": "integer" + }, + "laneId": { + "type": "string", + "pattern": "^[A-Z][A-Z0-9_-]{0,15}$" + }, + "entrant": { + "type": "object", + "additionalProperties": false, + "required": [ + "model", + "runtime", + "processType" + ], + "properties": { + "model": { + "type": "string", + "minLength": 1 + }, + "runtime": { + "type": "string", + "minLength": 1 + }, + "processType": { + "type": "string", + "enum": [ + "naked-model", + "workflow-system", + "scripted-agent", + "human-operated", + "other" + ] + } + } + }, + "protocolFreeze": { + "type": "object", + "additionalProperties": false, + "required": [ + "changedAfterLiveResults", + "scorerMutationObserved", + "calibrationOnlyIfChanged" + ], + "properties": { + "changedAfterLiveResults": { + "type": "boolean" + }, + "scorerMutationObserved": { + "type": "boolean" + }, + "calibrationOnlyIfChanged": { + "type": "boolean" + } + } + }, + "mechanical": { + "$ref": "#/$defs/mechanicalResult" + }, + "visual": { + "$ref": "#/$defs/visualResult" + }, + "workflow": { + "$ref": "#/$defs/workflowResult" + }, + "evidence": { + "$ref": "#/$defs/evidenceResult" + }, + "claimBoundary": { + "type": "string", + "enum": [ + "calibration-only", + "no-support", + "directional-signal", + "repeatable-workflow-value-support-candidate", + "public-benchmark-support" + ] + }, + "warnings": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "$defs": { + "mechanicalResult": { + "type": "object", + "additionalProperties": false, + "required": [ + "ranked", + "protocolVersion", + "passCount", + "totalAcs", + "determinism", + "failedAcs", + "hiddenChallengeSummary", + "regressionSummary", + "resultJsonRef" + ], + "properties": { + "ranked": { + "type": "boolean" + }, + "protocolVersion": { + "const": "2000m.driver.v3" + }, + "passCount": { + "type": "integer", + "minimum": 0 + }, + "totalAcs": { + "type": "integer", + "minimum": 1 + }, + "compositeScore": { + "type": "number", + "minimum": 0, + "maximum": 100 + }, + "determinism": { + "type": "object", + "additionalProperties": false, + "required": [ + "pass" + ], + "properties": { + "pass": { + "type": "boolean" + }, + "details": { + "type": "string" + } + } + }, + "failedAcs": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + } + }, + "hiddenChallengeSummary": { + "type": "string" + }, + "regressionSummary": { + "type": "string" + }, + "resultJsonRef": { + "type": "string", + "minLength": 1 + } + } + }, + "visualResult": { + "type": "object", + "additionalProperties": false, + "required": [ + "ranked", + "blockReason", + "visualPackageRef", + "captureDeterminism", + "rubricRecordRef" + ], + "properties": { + "ranked": { + "type": "boolean" + }, + "blockReason": { + "type": "string", + "enum": [ + "none", + "missing-native-capture-or-playable-surface", + "capture-command-failed", + "missing-capture-metadata", + "non-deterministic-capture", + "visual-seeds-mutated-after-live-results", + "blind-label-map-opened-before-rating", + "private-local-ref-in-public-record", + "asset-boundary-failed" + ] + }, + "visualPackageRef": { + "type": "string" + }, + "captureDeterminism": { + "type": "string", + "enum": [ + "passed", + "failed", + "not-run", + "blocked" + ] + }, + "rubricRecordRef": { + "type": "string" + }, + "score": { + "type": "number", + "minimum": 0, + "maximum": 100 + } + }, + "allOf": [ + { + "if": { + "properties": { + "ranked": { + "const": true + } + }, + "required": [ + "ranked" + ] + }, + "then": { + "properties": { + "blockReason": { + "const": "none" + }, + "visualPackageRef": { + "type": "string", + "minLength": 1 + }, + "captureDeterminism": { + "const": "passed" + }, + "rubricRecordRef": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "visualPackageRef", + "rubricRecordRef" + ] + } + }, + { + "if": { + "properties": { + "blockReason": { + "const": "none" + } + }, + "required": [ + "blockReason" + ] + }, + "then": { + "properties": { + "ranked": { + "const": true + }, + "visualPackageRef": { + "type": "string", + "minLength": 1 + }, + "captureDeterminism": { + "const": "passed" + }, + "rubricRecordRef": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "visualPackageRef", + "rubricRecordRef" + ] + } + } + ] + }, + "workflowResult": { + "type": "object", + "additionalProperties": false, + "required": [ + "contextWipeRecoveryScore", + "feedbackDecisionScore", + "regressionProtectionScore", + "impossibleRequirementHandlingScore", + "handoffScore", + "finalRecommendation", + "rationaleRefs" + ], + "properties": { + "contextWipeRecoveryScore": { + "type": "number", + "minimum": 0, + "maximum": 100 + }, + "feedbackDecisionScore": { + "type": "number", + "minimum": 0, + "maximum": 100 + }, + "regressionProtectionScore": { + "type": "number", + "minimum": 0, + "maximum": 100 + }, + "impossibleRequirementHandlingScore": { + "type": "number", + "minimum": 0, + "maximum": 100 + }, + "handoffScore": { + "type": "number", + "minimum": 0, + "maximum": 100 + }, + "finalRecommendation": { + "type": "string", + "enum": [ + "continue", + "stop", + "redesign", + "inspect_scorer" + ] + }, + "rationaleRefs": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + } + } + } + }, + "evidenceResult": { + "type": "object", + "additionalProperties": false, + "required": [ + "replayable", + "publicSafe", + "privateRefsBlocked", + "compactSummaryRef", + "requiredRefsMissing", + "claimBoundary" + ], + "properties": { + "replayable": { + "type": "boolean" + }, + "publicSafe": { + "type": "boolean" + }, + "privateRefsBlocked": { + "type": "boolean" + }, + "compactSummaryRef": { + "type": "string" + }, + "requiredRefsMissing": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + } + }, + "claimBoundary": { + "type": "string", + "enum": [ + "calibration-only", + "no-support", + "directional-signal", + "repeatable-workflow-value-support-candidate", + "public-benchmark-support" + ] + } + } + } + } +} diff --git a/v3/run-record.schema.json b/v3/run-record.schema.json new file mode 100644 index 0000000..4a252ce --- /dev/null +++ b/v3/run-record.schema.json @@ -0,0 +1,570 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/graphanov/2000m/v3/run-record.schema.json", + "title": "2000m v3 generic run record", + "type": "object", + "additionalProperties": false, + "required": [ + "schemaVersion", + "campaignId", + "scenarioId", + "taskSeed", + "laneId", + "entrant", + "artifact", + "protocolFreeze", + "phases", + "mechanical", + "visual", + "workflow", + "evidence", + "finalRecommendation" + ], + "properties": { + "schemaVersion": { + "const": "2000m.v3.run-record.v1" + }, + "campaignId": { + "type": "string", + "pattern": "^[a-z0-9][a-z0-9-]{2,100}$" + }, + "scenarioId": { + "type": "string", + "pattern": "^[a-z0-9][a-z0-9-]{2,100}$" + }, + "taskSeed": { + "type": "integer" + }, + "laneId": { + "type": "string", + "pattern": "^[A-Z][A-Z0-9_-]{0,15}$" + }, + "entrant": { + "type": "object", + "additionalProperties": false, + "required": [ + "model", + "runtime", + "processType" + ], + "properties": { + "model": { + "type": "string", + "minLength": 1 + }, + "runtime": { + "type": "string", + "minLength": 1 + }, + "processType": { + "type": "string", + "enum": [ + "naked-model", + "workflow-system", + "scripted-agent", + "human-operated", + "other" + ] + }, + "notes": { + "type": "string" + } + } + }, + "artifact": { + "type": "object", + "additionalProperties": false, + "required": [ + "repoOrPath", + "commitOrDigest", + "manifestRef" + ], + "properties": { + "repoOrPath": { + "type": "string", + "minLength": 1 + }, + "commitOrDigest": { + "type": "string", + "minLength": 1 + }, + "manifestRef": { + "type": "string", + "minLength": 1 + }, + "changedFilesRef": { + "type": "string" + } + } + }, + "protocolFreeze": { + "type": "object", + "additionalProperties": false, + "required": [ + "frozen", + "frozenCampaignRef", + "changedAfterLiveResults", + "scorerMutationObserved" + ], + "properties": { + "frozen": { + "type": "boolean" + }, + "frozenCampaignRef": { + "type": "string", + "minLength": 1 + }, + "changedAfterLiveResults": { + "type": "boolean" + }, + "scorerMutationObserved": { + "type": "boolean" + }, + "calibrationReason": { + "type": "string" + } + } + }, + "phases": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/$defs/phaseRecord" + } + }, + "mechanical": { + "$ref": "#/$defs/mechanicalStatus" + }, + "visual": { + "$ref": "#/$defs/visualStatus" + }, + "workflow": { + "$ref": "#/$defs/workflowStatus" + }, + "evidence": { + "$ref": "#/$defs/evidenceStatus" + }, + "finalRecommendation": { + "$ref": "#/$defs/finalRecommendation" + } + }, + "$defs": { + "phaseRecord": { + "type": "object", + "additionalProperties": false, + "required": [ + "phaseId", + "kind", + "outputs", + "evidenceRefs" + ], + "properties": { + "phaseId": { + "type": "string", + "pattern": "^[a-z0-9][a-z0-9-]{1,80}$" + }, + "kind": { + "type": "string", + "enum": [ + "protocol-freeze", + "initial-attempt", + "context-wipe-recovery", + "reviewer-injection", + "regression-trap", + "stale-or-impossible-requirement", + "final-handoff" + ] + }, + "outputs": { + "type": "object" + }, + "feedbackDecisions": { + "type": "array", + "items": { + "$ref": "#/$defs/feedbackDecision" + } + }, + "evidenceRefs": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "label", + "ref", + "kind" + ], + "properties": { + "label": { + "type": "string", + "minLength": 1 + }, + "ref": { + "type": "string", + "minLength": 1 + }, + "kind": { + "type": "string", + "enum": [ + "repo", + "commit", + "manifest", + "driver-protocol", + "mechanical-ac-spec", + "visual-rubric", + "scoring-model", + "sandbox-policy", + "scenario", + "campaign", + "prompt", + "model-response", + "stdout", + "stderr", + "log", + "conformance-json", + "capture", + "screenshot", + "gif", + "webm", + "frames", + "rubric-record", + "summary", + "other" + ] + } + } + } + } + } + }, + "feedbackDecision": { + "type": "object", + "additionalProperties": false, + "required": [ + "feedbackId", + "decision", + "rationale", + "evidenceRef" + ], + "properties": { + "feedbackId": { + "type": "string", + "minLength": 1 + }, + "decision": { + "type": "string", + "enum": [ + "accepted", + "rejected_with_reason", + "deferred", + "needs_scorer_inspection", + "needs_owner_taste_gate" + ] + }, + "rationale": { + "type": "string", + "minLength": 1 + }, + "evidenceRef": { + "type": "string", + "minLength": 1 + } + } + }, + "mechanicalStatus": { + "type": "object", + "additionalProperties": false, + "required": [ + "ranked", + "protocolVersion", + "resultJsonRef" + ], + "properties": { + "ranked": { + "type": "boolean" + }, + "protocolVersion": { + "const": "2000m.driver.v3" + }, + "resultJsonRef": { + "type": "string", + "minLength": 1 + }, + "passCount": { + "type": "integer", + "minimum": 0 + }, + "totalAcs": { + "type": "integer", + "minimum": 0 + }, + "failedAcs": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + } + } + } + }, + "visualStatus": { + "type": "object", + "additionalProperties": false, + "required": [ + "ranked", + "blockReason", + "visualPackageRef" + ], + "properties": { + "ranked": { + "type": "boolean" + }, + "blockReason": { + "type": "string", + "enum": [ + "none", + "missing-native-capture-or-playable-surface", + "capture-command-failed", + "missing-capture-metadata", + "non-deterministic-capture", + "visual-seeds-mutated-after-live-results", + "blind-label-map-opened-before-rating", + "private-local-ref-in-public-record", + "asset-boundary-failed" + ] + }, + "visualPackageRef": { + "type": "string" + }, + "captureCommandResultRef": { + "type": "string" + } + }, + "allOf": [ + { + "if": { + "properties": { + "ranked": { + "const": true + } + }, + "required": [ + "ranked" + ] + }, + "then": { + "properties": { + "blockReason": { + "const": "none" + }, + "visualPackageRef": { + "type": "string", + "minLength": 1 + }, + "captureCommandResultRef": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "visualPackageRef", + "captureCommandResultRef" + ] + } + }, + { + "if": { + "properties": { + "blockReason": { + "const": "none" + } + }, + "required": [ + "blockReason" + ] + }, + "then": { + "properties": { + "ranked": { + "const": true + }, + "visualPackageRef": { + "type": "string", + "minLength": 1 + }, + "captureCommandResultRef": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "visualPackageRef", + "captureCommandResultRef" + ] + } + } + ] + }, + "workflowStatus": { + "type": "object", + "additionalProperties": false, + "required": [ + "contextWipeStatus", + "feedbackDecisionStatus", + "regressionStatus", + "handoffStatus" + ], + "properties": { + "contextWipeStatus": { + "type": "string", + "enum": [ + "not-reached", + "passed", + "failed", + "blocked" + ] + }, + "feedbackDecisionStatus": { + "type": "string", + "enum": [ + "not-reached", + "passed", + "failed", + "blocked" + ] + }, + "regressionStatus": { + "type": "string", + "enum": [ + "not-reached", + "passed", + "failed", + "blocked" + ] + }, + "handoffStatus": { + "type": "string", + "enum": [ + "not-reached", + "passed", + "failed", + "blocked" + ] + } + } + }, + "evidenceStatus": { + "type": "object", + "additionalProperties": false, + "required": [ + "publicSafe", + "privateRefsBlocked", + "requiredRefsMissing", + "claimBoundary", + "refs" + ], + "properties": { + "publicSafe": { + "type": "boolean" + }, + "privateRefsBlocked": { + "type": "boolean" + }, + "requiredRefsMissing": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + } + }, + "claimBoundary": { + "type": "string", + "enum": [ + "calibration-only", + "no-support", + "directional-signal", + "repeatable-workflow-value-support-candidate", + "public-benchmark-support" + ] + }, + "refs": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "label", + "ref", + "kind" + ], + "properties": { + "label": { + "type": "string", + "minLength": 1 + }, + "ref": { + "type": "string", + "minLength": 1 + }, + "kind": { + "type": "string", + "enum": [ + "repo", + "commit", + "manifest", + "driver-protocol", + "mechanical-ac-spec", + "visual-rubric", + "scoring-model", + "sandbox-policy", + "scenario", + "campaign", + "prompt", + "model-response", + "stdout", + "stderr", + "log", + "conformance-json", + "capture", + "screenshot", + "gif", + "webm", + "frames", + "rubric-record", + "summary", + "other" + ] + } + } + } + } + } + }, + "finalRecommendation": { + "type": "object", + "additionalProperties": false, + "required": [ + "decision", + "rationaleRefs" + ], + "properties": { + "decision": { + "type": "string", + "enum": [ + "continue", + "stop", + "redesign", + "inspect_scorer" + ] + }, + "rationaleRefs": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + } + } + } + } + } +} diff --git a/v3/visual-package.schema.json b/v3/visual-package.schema.json new file mode 100644 index 0000000..7e0f3e4 --- /dev/null +++ b/v3/visual-package.schema.json @@ -0,0 +1,199 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/graphanov/2000m/v3/visual-package.schema.json", + "title": "2000m v3 visual package", + "type": "object", + "additionalProperties": false, + "required": [ + "schemaVersion", + "packageId", + "campaignId", + "scenarioId", + "taskSeed", + "anonymized", + "mappingSealedBeforeReview", + "artifactDigest", + "windows", + "artifacts", + "rubric" + ], + "properties": { + "schemaVersion": { + "const": "2000m.v3.visual-package.v1" + }, + "packageId": { + "type": "string", + "pattern": "^[a-z0-9][a-z0-9-]{2,100}$" + }, + "campaignId": { + "type": "string", + "pattern": "^[a-z0-9][a-z0-9-]{2,100}$" + }, + "scenarioId": { + "type": "string", + "pattern": "^[a-z0-9][a-z0-9-]{2,100}$" + }, + "taskSeed": { + "type": "integer" + }, + "anonymized": { + "type": "boolean" + }, + "mappingSealedBeforeReview": { + "type": "boolean" + }, + "artifactDigest": { + "type": "string", + "pattern": "^sha256:[0-9a-f]{64}$" + }, + "windows": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/$defs/captureWindow" + } + }, + "artifacts": { + "type": "object", + "additionalProperties": false, + "required": [ + "blindContactSheetRef", + "blindLabelMapRef", + "screenshotsDirRef", + "replaysDirRef", + "framesDirRef" + ], + "properties": { + "blindContactSheetRef": { + "type": "string", + "minLength": 1 + }, + "blindLabelMapRef": { + "type": "string", + "minLength": 1 + }, + "screenshotsDirRef": { + "type": "string", + "minLength": 1 + }, + "replaysDirRef": { + "type": "string", + "minLength": 1 + }, + "framesDirRef": { + "type": "string", + "minLength": 1 + } + } + }, + "rubric": { + "type": "object", + "additionalProperties": false, + "required": [ + "rubricRef", + "rubricRecordRef", + "reviewerCount", + "ownerTasteSeparated" + ], + "properties": { + "rubricRef": { + "type": "string", + "minLength": 1 + }, + "rubricRecordRef": { + "type": "string", + "minLength": 1 + }, + "reviewerCount": { + "type": "integer", + "minimum": 1 + }, + "ownerTasteSeparated": { + "type": "boolean" + } + } + } + }, + "$defs": { + "captureWindow": { + "type": "object", + "additionalProperties": false, + "required": [ + "window", + "seed", + "captureCommand", + "screenshotRef", + "replayRef", + "frameMetadataRef", + "rubricMetadataRef", + "fps", + "frameCount", + "inputSequenceRef", + "stateChecksum", + "frameChecksum", + "warnings" + ], + "properties": { + "window": { + "type": "string", + "enum": [ + "early-game", + "mid-run-obstacle-field", + "ramp-style-sequence", + "monster-pressure", + "post-feedback-rerun" + ] + }, + "seed": { + "type": "integer" + }, + "captureCommand": { + "type": "string", + "minLength": 1 + }, + "screenshotRef": { + "type": "string", + "minLength": 1 + }, + "replayRef": { + "type": "string", + "minLength": 1 + }, + "frameMetadataRef": { + "type": "string", + "minLength": 1 + }, + "rubricMetadataRef": { + "type": "string", + "minLength": 1 + }, + "fps": { + "type": "integer", + "minimum": 1 + }, + "frameCount": { + "type": "integer", + "minimum": 1 + }, + "inputSequenceRef": { + "type": "string", + "minLength": 1 + }, + "stateChecksum": { + "type": "string", + "pattern": "^sha256:[0-9a-f]{64}$" + }, + "frameChecksum": { + "type": "string", + "pattern": "^sha256:[0-9a-f]{64}$" + }, + "warnings": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + } +}