diff --git a/docs/apps-module-migrations.json b/docs/apps-module-migrations.json index 9090a5956..503abf3be 100644 --- a/docs/apps-module-migrations.json +++ b/docs/apps-module-migrations.json @@ -36,6 +36,7 @@ "tests/test_apps_prepare.py", "tests/test_apps_publication.py", "tests/test_apps_replay.py", + "tests/test_benchmark_preparation.py", "tests/test_result_delivery.py" ] }, @@ -137,7 +138,8 @@ "tests/test_apps_evidence.py", "tests/test_apps_execution_binding.py", "tests/test_apps_generation_plan.py", - "tests/test_apps_module_migrations.py" + "tests/test_apps_module_migrations.py", + "tests/test_benchmark_preparation.py" ] }, { diff --git a/docs/contracts/production-prepare.md b/docs/contracts/production-prepare.md new file mode 100644 index 000000000..f87ff1801 --- /dev/null +++ b/docs/contracts/production-prepare.md @@ -0,0 +1,99 @@ +# Production P1 preparation + +`inferrsi.benchmark.preparation.PreparationService.prepare` implements the +Benchmark preparation port for the accepted D023 / `t108-v4` task family: +`p1-fixture@1`, `p1-rules@1`, operation `evaluate`, scope `p1`. + +The trusted host configures a **Benchmark-owned frozen ProtocolV1** and a +read-only resource capability. Neither is supplied inside PrepareRequest. +The task configuration includes an explicit resource role for the actual worker +code bytes. Its ArtifactRef digest must equal `worker_recipe.code_sha256`. +Preparation verifies the exact supplied bytes; it never treats the historical +documentation worker digest as an available executable. + +```python +from inferrsi.benchmark.preparation import ( + PreparationService, encode, decode, validate_prepared, +) + +service = PreparationService( + frozen_protocol, resources=open_fixed_resource, worker_role="worker-code", +) +prepared, protocol_bytes, open_resource = service.prepare(request) +``` + +`open_fixed_resource(role)` is a host-supplied capability returning bytes or a +fresh binary stream for an authorized role. Preparation closes each stream it +opens for verification; after prepare, the closure's consumer (normally Apps) +owns closing each stream it opens. The opener does not interpret a +request-provided location as a filesystem path. +The frozen resource metadata is verified during preparation. The returned +`open_resource` accepts only roles in that preparation's closure and reopens the +resource through the same capability. It does not write or archive bytes. +Provisional source locations may alias: byte authority is the configured role, +not a lookup or de-duplication by that location. Apps assigns a distinct final +archive path to each role and verifies the bytes against that role's metadata. + +The first task keeps the P1 candidate, claims, evidence rules, worker recipe +shape and sampling conditions. PrepareRequest can bind its candidate expectation, +P1 task parameters, budget references and additional resource pins. A supplied +pin cannot replace a frozen role's hash, format or required use. The first task +does allow that pin to replace the role's provisional source location; this +changes the frozen protocol bytes, with identity effects described below. The first task +accepts an empty `comparison_conditions`: comparison conditions are configured +in the frozen task definition, rather than silently accepting unknown overrides. +Unknown task, grader, worker or claim-rule revisions are refused. + +The protocol contains its declared resources and optional baseline/reference/ +quality-reference resources. PreparedEvaluation carries the complete closure, +sorted by unique role. Duplicate roles across any of these positions are refused. + +## Identity and pure codecs + +`encode(prepared)`, `decode(bytes)` and `validate_prepared(prepared, +protocol_bytes)` perform no artifact reads, worker imports, execution or writes. +They preserve the canonical `inferrsi-json-v1` byte contract. + +- `prepared_id` excludes its own value, `created_at`, `display_notes`, and + locations in its schema-declared outer ArtifactRefs. Moving both the outer + protocol and outer resource references preserves this ID. +- `protocol_digest` excludes protocol display notes and locations only in the + declared resource references, including baseline/reference/quality-reference. + It preserves an arbitrary ref-shaped value inside Parameters. +- Moving a reference **inside** the protocol changes the protocol's canonical + bytes and byte digest. The changed digest changes `prepared_id`, even when + `protocol_digest` remains equal. Other semantic changes affect both identities. + +## Apps boundary and failures + +Inject the bound port and pure functions into the existing Apps API: + +```python +from inferrsi.apps.api import prepare + +output = prepare( + {"schema_version": 1, "request": request}, + workspace=workspace, storage_config=storage_config, + prepare_benchmark=service.prepare, + encode_prepared=encode, validate_prepared=validate_prepared, +) +``` + +Apps assigns its final outer locations, invokes the Benchmark pure codecs, +verifies bytes independently, then commits the full closure before publishing +a PreparedRef. Benchmark imports neither Apps nor Harness. + +The prepare port returns a portable PublicError wire object for a declared +rejection. Missing resources produce `ArtifactUnavailable` at `prepare`; +incorrect retained metadata produces `ArtifactIntegrityError`. No preparation +is archived by Benchmark. If the resource disappears after preparation, the +Apps archive reports `ArtifactUnavailable` at `prepared_archive`; changed bytes +are rejected by its independent integrity verifier. Neither case publishes a +PreparedRef or a partial prepared closure. + +This port does not register the configured worker with a Harness target or +execute/assess it. The H evaluation-mode path and production offline assessor +remain separate deliveries under #429 and #434. The packaged engineering-task +registry is unchanged. A host must supply the reviewed P1 task definition and +actual worker/resource capability; no default fixture backend is silently +selected. Ordinary CPU regressions do not establish a model/service gain. diff --git a/docs/operations.md b/docs/operations.md index 1e1842786..db301c7ae 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -51,6 +51,7 @@ Effect classes used below: | Rebuild scoped legacy evidence views | `python tools/rebuild_model_evidence.py --archive --output ` | CPU-only; creates a new output directory | Authenticates original bytes and pinned projection/code; no extraction, network, archived-code execution or campaign qualification | | Inspect evidence inventory / generate budget input | `python tools/validate_evidence.py [--budget-assets]` | Read-only | Offline layout/hash/schema checks; optional disposable #328 metadata view derived from campaign authority; no downloads | | Execute one ordinary local CPU command with a fixed record | Python `inferrsi.harness.api.Harness.allocate` / `execute` / `read` | Experiment-executing / record-writing | One explicit RunStore root and one trusted local CPU target; ordinary command mode only; a record holds execution facts, so exit 0 is not a verdict; no device, model or network fallback | +| Freeze a configured P1 preparation | Python `benchmark.preparation.PreparationService.prepare`; [production prepare](contracts/production-prepare.md) | Read-only resource access / pure encoding | Verifies supplied frozen worker/resource bytes and returns a preparation, protocol and role-keyed closure; Apps separately archives them before publishing a Ref | | Visualize a recorded run or campaign | CLI `dashboard`; [dashboard](dashboard.md) | Report-generating / local read-only serving | Writes an offline HTML snapshot or serves an optional live view on loopback; does not run an agent or validate a claim | | Capture a knowledge context | CLI `campaign capture-context`; [offline dashboard](dashboard.md#record-the-optimization-story-campaign-trace) | Local evidence-writing | Saves assembled context and resource hashes in a new `knowledge/.json`. Supports ordinary questions and explicit IDs. The only accepted explicit `--relevance-method` value is `find-lexical-v1` | | Retain an executed retrieval | Python `apps.workflows.campaign.save_retrieval`; [capture contract](knowledge-retrieval-trace.md#retain-a-standalone-result) | Local evidence-writing | Saves an existing catalog/collect result under a fresh campaign capture ID; no new search or automatic publication | diff --git a/src/inferrsi/benchmark/preparation.py b/src/inferrsi/benchmark/preparation.py new file mode 100644 index 000000000..ddcd5ec0a --- /dev/null +++ b/src/inferrsi/benchmark/preparation.py @@ -0,0 +1,451 @@ +"""Frozen D023/P1 preparations over a Benchmark-owned task and byte capability. + +The host configures the frozen task and a role-keyed read-only resource opener. +PrepareRequest cannot supply task rules, worker code or a filesystem authority. +Preparation neither executes the worker nor persists a preparation or its bytes. +""" + +from __future__ import annotations + +import re +from collections.abc import Callable +from contextlib import closing +from copy import deepcopy +from datetime import datetime +from hashlib import sha256 +from pathlib import PurePosixPath + +from ..contracts.json_v1 import CanonicalError +from ..contracts.json_v1 import decode as canonical_decode +from ..contracts.json_v1 import encode as canonical_encode + +_REQUEST = set("schema_version task operation validation_scope input_pins task_parameters " + "comparison_conditions expected_candidate budget_policy_refs".split()) +_PROTOCOL = set("schema_version task grader operation validation_scope candidate_contract " + "worker_recipe input_requirements claim_definitions metric_definitions " + "required_evidence task_parameters baseline reference quality_reference " + "sampling feedback_visibility budget_policy_refs resources display_notes".split()) +_PREPARED = set("schema_version prepared_id task grader operation validation_scope protocol " + "protocol_digest candidate_contract expected_candidate required_evidence " + "recipe resources created_at display_notes".split()) +_COPIES = ("task", "grader", "operation", "validation_scope", "candidate_contract", + "required_evidence") +_RESOURCE = {"role", "artifact", "format_schema", "required_for"} +_ARTIFACT = {"location", "sha256", "size_bytes", "media_type"} +_ID = re.compile(r"[A-Za-z0-9][A-Za-z0-9._:/@+\-]{0,255}\Z") +_SHA = re.compile(r"[a-f0-9]{64}\Z") +_DECIMAL = re.compile(r"(?:0|-?[1-9][0-9]*|-?(?:0|[1-9][0-9]*)\.[0-9]*[1-9])\Z") +_CANDIDATE = { + "kind": "text", "input_role": "candidate", + "format_schema": {"name": "utf8-text", "revision": "1"}, + "identity_rule": {"name": "captured-sha256", "revision": "1"}, + "entrypoint": None, "required_paths": [], "allowed_content": {}, + "modification_receipt_required": False, +} + + +class PreparationError(ValueError): + """A Benchmark-owned rejection, with the portable PublicError wire form.""" + + def __init__(self, code: str, message: str): + super().__init__(message) + self.code = code + + def to_json(self) -> dict: + return { + "code": self.code, "stage": "prepare", "attempt_id": None, "started": None, + "latest_record_ref": None, "partial_record": None, "retained_refs": [], + "field_errors": [], "message": str(self), + } + + +def _require(condition, message: str, code: str = "InvalidRequest") -> None: + if not condition: + raise PreparationError(code, message) + + +def _object(value, fields) -> None: + _require(isinstance(value, dict) and value.keys() == fields, "Unexpected object fields.") + + +def _id(value) -> bool: + return isinstance(value, str) and _ID.fullmatch(value) is not None + + +def _definition(value) -> None: + _object(value, {"name", "revision"}) + _require(all(_id(item) for item in value.values()), "Invalid definition reference.") + + +def _artifact(value) -> None: + _object(value, _ARTIFACT) + location = value["location"] + _require(isinstance(location, str) and bool(location) and "\\" not in location + and ":" not in location and not location.startswith("/") + and all(part not in ("", ".", "..") for part in location.split("/")) + and str(PurePosixPath(location)) == location, "Invalid logical artifact location.") + _require(isinstance(value["sha256"], str) and _SHA.fullmatch(value["sha256"]) is not None + and type(value["size_bytes"]) is int and value["size_bytes"] >= 0 + and isinstance(value["media_type"], str) and bool(value["media_type"]), + "Invalid artifact metadata.") + + +def _resource(value) -> None: + _object(value, _RESOURCE) + _require(_id(value["role"]), "Invalid resource role.") + _artifact(value["artifact"]) + _definition(value["format_schema"]) + uses = value["required_for"] + _require(isinstance(uses, list) and bool(uses) + and all(isinstance(use, str) for use in uses) + and len(uses) == len(set(uses)) and set(uses) <= {"prepare", "execute", "assess"}, + "Invalid resource use.") + + +def _closure(protocol: dict) -> list[dict]: + _require(isinstance(protocol["resources"], list), "Expected protocol resources.") + values = list(protocol["resources"]) + values += [protocol[key] for key in ("baseline", "reference", "quality_reference") + if protocol[key] is not None] + for value in values: + _resource(value) + roles = [value["role"] for value in values] + _require(len(roles) == len(set(roles)), "Resource roles must be unique across the closure.") + return sorted(values, key=lambda value: value["role"]) + + +def _project_resource(value: dict) -> dict: + _resource(value) + return {**value, "artifact": {key: item for key, item in value["artifact"].items() + if key != "location"}} + + +def protocol_digest(protocol: dict) -> str: + """Project declared ArtifactRefs only; ref-shaped Parameters stay semantic.""" + _object(protocol, _PROTOCOL) + value = deepcopy(protocol) + value.pop("display_notes") + value["resources"] = [_project_resource(item) for item in value["resources"]] + for key in ("baseline", "reference", "quality_reference"): + if value[key] is not None: + value[key] = _project_resource(value[key]) + return "sha256:" + sha256(canonical_encode(value)).hexdigest() + + +def prepared_id(prepared: dict) -> str: + """Compute identity independently of outer locators, clock and display notes.""" + _object(prepared, _PREPARED) + value = deepcopy(prepared) + for key in ("prepared_id", "created_at", "display_notes"): + value.pop(key) + _artifact(value["protocol"]) + value["protocol"].pop("location") + value["resources"] = [_project_resource(item) for item in value["resources"]] + return "sha256:" + sha256(canonical_encode(value)).hexdigest() + + +def _parameters(value) -> None: + _object(value, {"decimal_fixture", "label"}) + _require(isinstance(value["decimal_fixture"], str) + and _DECIMAL.fullmatch(value["decimal_fixture"]) is not None + and isinstance(value["label"], str), "Invalid P1 task parameters.") + + +def _candidate(value) -> None: + _require(canonical_encode(value) == canonical_encode(_CANDIDATE), + "Unsupported P1 candidate contract.", "UnsupportedRuleVersion") + + +def _recipe(recipe) -> None: + _object(recipe, set("recipe_revision runtime_role entrypoint_kind entrypoint " + "entrypoint_revision code_sha256 args input_roles outputs " + "required_capabilities".split())) + fixed = {"recipe_revision": "1", "runtime_role": "local-python", + "entrypoint_kind": "python_module", "entrypoint": "fixture_worker", + "entrypoint_revision": "1"} + _require(all(recipe[key] == value for key, value in fixed.items()), + "Unsupported frozen P1 worker revision.", "UnsupportedRuleVersion") + _require(isinstance(recipe["code_sha256"], str) + and _SHA.fullmatch(recipe["code_sha256"]) is not None, + "Invalid frozen worker digest.") + for key in ("args", "input_roles", "required_capabilities"): + _require(isinstance(recipe[key], list) + and all(isinstance(item, str) and item for item in recipe[key]), + "Invalid worker arguments or roles.") + _require(recipe["input_roles"] == ["candidate"] + and recipe["args"] == ["{input:candidate}", "{output:grader}"] + and recipe["required_capabilities"] == ["local-process-tree"], + "Unsupported P1 worker recipe.", "UnsupportedRuleVersion") + _require(recipe["outputs"] == [{ + "role": "grader", "path": "outputs/grader.json", "media_type": "application/json", + "format_schema": {"name": "fixture-grader", "revision": "1"}, "required": True, + }] and type(recipe["outputs"][0]["required"]) is bool, + "Unsupported P1 worker output.", "UnsupportedRuleVersion") + + +def _requirements(requirements) -> None: + _require(isinstance(requirements, list) and len(requirements) == 4, + "Expected the P1 evidence requirements.") + expected = { + "format": ("F", "preparing", {"harness_capture"}), + "grader": ("C", "finalizing", {"task_worker"}), + "samples": ("M", "running", {"harness_capture", "task_worker"}), + "cleanup": ("M", "finalizing", {"harness_capture"}), + } + for item in requirements: + _object(item, set("check_id checker_revision allowed_producer_kinds subject phase " + "artifact_schema matching_rule_revision allowed_control_domains " + "for_claims".split())) + key = item["check_id"] + _require(isinstance(key, str) and key in expected, "Unsupported P1 check.", + "UnsupportedRuleVersion") + claim, phase, producers = expected.pop(key) + kinds, domains = item["allowed_producer_kinds"], item["allowed_control_domains"] + _require(isinstance(kinds, list) and all(_id(kind) for kind in kinds) + and len(kinds) == len(set(kinds)) and set(kinds) == producers + and isinstance(domains, list) and bool(domains) + and all(_id(domain) for domain in domains) + and len(domains) == len(set(domains)) + and item["checker_revision"] == item["matching_rule_revision"] == "1" + and item["subject"] == "candidate" and item["phase"] == phase + and item["for_claims"] == [claim] + and item["artifact_schema"] == {"name": key + "-proof", "revision": "1"}, + "Unsupported frozen P1 evidence rule.", "UnsupportedRuleVersion") + + +def _protocol(protocol: dict) -> None: + canonical_encode(protocol) + _object(protocol, _PROTOCOL) + _require(type(protocol["schema_version"]) is int and protocol["schema_version"] == 1, + "Unsupported protocol version.", "UnsupportedSchema") + _require(protocol["task"] == {"name": "p1-fixture", "revision": "1"} + and protocol["grader"] == {"name": "p1-rules", "revision": "1"}, + "Only the accepted P1 task and grader revision are supported.", + "UnsupportedRuleVersion") + _require(protocol["operation"] == "evaluate" and protocol["validation_scope"] == "p1", + "Unsupported task operation or scope.", "UnsupportedOperation") + _parameters(protocol["task_parameters"]) + _closure(protocol) + _candidate(protocol["candidate_contract"]) + _recipe(protocol["worker_recipe"]) + _requirements(protocol["required_evidence"]) + _require(protocol["input_requirements"] == {"candidate_role": "candidate"} + and protocol["metric_definitions"] == [] + and protocol["sampling"] == {"order": ["AB", "BA"], "repetitions": 2} + and type(protocol["sampling"]["repetitions"]) is int + and protocol["feedback_visibility"] == ["format"] + and isinstance(protocol["display_notes"], str), + "Unsupported P1 protocol conditions.", "UnsupportedRuleVersion") + claims = protocol["claim_definitions"] + _require(isinstance(claims, list) and len(claims) == 3, "Expected the P1 claim registry.") + expected = {"F": ["format"], "C": ["grader"], "M": ["samples", "cleanup"]} + for claim in claims: + _object(claim, {"claim_id", "validation_scope", "decision_rule", "required_checks", + "requires_completed_measurement", "requires_cleanup_confirmed"}) + key = claim["claim_id"] + _require(isinstance(key, str) and key in expected + and claim["decision_rule"] == {"name": "p1-" + key, "revision": "1"} + and claim["validation_scope"] == "p1" + and claim["required_checks"] == expected.pop(key) + and type(claim["requires_completed_measurement"]) is bool + and type(claim["requires_cleanup_confirmed"]) is bool + and claim["requires_completed_measurement"] == (key == "M") + and claim["requires_cleanup_confirmed"] == (key == "M"), + "Unsupported frozen P1 claim rule.", "UnsupportedRuleVersion") + _require(isinstance(protocol["budget_policy_refs"], list), "Expected budget references.") + for ref in protocol["budget_policy_refs"]: + _definition(ref) + + +def _expectation(candidate, contract: dict) -> None: + if candidate is None: + return + _object(candidate, {"candidate_id", "sha256", "format_schema"}) + _require(_id(candidate["candidate_id"]) and isinstance(candidate["sha256"], str) + and _SHA.fullmatch(candidate["sha256"]) is not None, + "Invalid expected candidate.") + _definition(candidate["format_schema"]) + _require(candidate["format_schema"] == contract["format_schema"], + "Expected candidate format differs from the frozen contract.") + + +def _prepared(prepared: dict) -> None: + canonical_encode(prepared) + _object(prepared, _PREPARED) + _require(type(prepared["schema_version"]) is int and prepared["schema_version"] == 1, + "Unsupported prepared version.", "UnsupportedSchema") + _artifact(prepared["protocol"]) + _candidate(prepared["candidate_contract"]) + _recipe(prepared["recipe"]) + _requirements(prepared["required_evidence"]) + _require(prepared["task"] == {"name": "p1-fixture", "revision": "1"} + and prepared["grader"] == {"name": "p1-rules", "revision": "1"} + and prepared["operation"] == "evaluate" and prepared["validation_scope"] == "p1", + "Unsupported prepared task revision or scope.", "UnsupportedRuleVersion") + _require(isinstance(prepared["display_notes"], str) + and isinstance(prepared["protocol_digest"], str) + and re.fullmatch(r"sha256:[a-f0-9]{64}", prepared["protocol_digest"]) is not None, + "Invalid preparation metadata.") + timestamp = prepared["created_at"] + if timestamp is not None: + _require(isinstance(timestamp, str) + and re.fullmatch(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,9})?Z", + timestamp) is not None, + "Invalid preparation timestamp.") + try: + datetime.fromisoformat(timestamp.replace("Z", "+00:00")) + except ValueError: + raise PreparationError("InvalidRequest", "Invalid preparation timestamp.") from None + _require(isinstance(prepared["resources"], list), "Expected prepared resources.") + for resource in prepared["resources"]: + _resource(resource) + roles = [resource["role"] for resource in prepared["resources"]] + _require(roles == sorted(set(roles)), "Prepared closure must be sorted by unique role.") + _expectation(prepared["expected_candidate"], prepared["candidate_contract"]) + _require(prepared["prepared_id"] == prepared_id(prepared), + "Prepared identity does not match its content.", "IdentityMismatch") + + +def encode(prepared: dict) -> bytes: + """Pure prepared encoding, including its self-identity check.""" + _prepared(prepared) + return canonical_encode(prepared) + + +def decode(data: bytes) -> dict: + """Decode canonical prepared bytes without resolving or opening any artifact.""" + value = canonical_decode(data) + _require(canonical_encode(value) == data, "Prepared bytes are not canonical.") + _prepared(value) + return value + + +def validate_prepared(prepared: dict, protocol_bytes: bytes) -> None: + """Pure validation of the full prepared/protocol identity and closure binding.""" + _prepared(prepared) + protocol = canonical_decode(protocol_bytes) + _require(canonical_encode(protocol) == protocol_bytes, "Protocol bytes are not canonical.") + _protocol(protocol) + ref = prepared["protocol"] + _require(ref["sha256"] == sha256(protocol_bytes).hexdigest() + and ref["size_bytes"] == len(protocol_bytes) + and ref["media_type"] == "application/json" + and prepared["protocol_digest"] == protocol_digest(protocol) + and all(prepared[key] == protocol[key] for key in _COPIES) + and prepared["recipe"] == protocol["worker_recipe"] + and [_project_resource(item) for item in prepared["resources"]] + == [_project_resource(item) for item in _closure(protocol)], + "Prepared object disagrees with its frozen protocol.", "IdentityMismatch") + + +def _measure(source) -> tuple[int, str]: + if source is None: + raise FileNotFoundError("A required resource is unavailable.") + if isinstance(source, bytes): + return len(source), sha256(source).hexdigest() + size, digest = 0, sha256() + with closing(source): + while True: + chunk = source.read(1024 * 1024) + if not isinstance(chunk, bytes): + raise TypeError("A host resource capability must return binary bytes.") + if not chunk: + break + size += len(chunk) + digest.update(chunk) + return size, digest.hexdigest() + + +class PreparationService: + """The supported P1 prepare port bound to frozen rules and read capabilities. + + ``worker_role`` identifies the retained code bytes for recipe.code_sha256. + Preparation closes every stream it opens for verification. After prepare, + the consumer of the returned resource reader owns each stream it opens. + The returned closure reopens resources: deletion fails at the Apps archive + boundary and changed bytes are independently rejected by Apps' verifier. + """ + + def __init__(self, protocol: dict, *, resources: Callable, worker_role: str): + _protocol(protocol) + _require(callable(resources) and _id(worker_role), "Invalid resource capability.") + closure = {item["role"]: item for item in _closure(protocol)} + _require(worker_role in closure and closure[worker_role]["artifact"]["sha256"] + == protocol["worker_recipe"]["code_sha256"], + "The worker recipe must bind retained code bytes.", "IdentityMismatch") + self._frozen = canonical_encode(protocol) + self._resources = resources + + def prepare(self, request: dict) -> tuple | dict: + """Return the prepared object, frozen protocol bytes and read-only closure.""" + try: + canonical_encode(request) + except CanonicalError: + return PreparationError("InvalidRequest", "Malformed prepare request.").to_json() + try: + return self._prepare(request) + except PreparationError as error: + return error.to_json() + + def _prepare(self, request: dict) -> tuple: + _object(request, _REQUEST) + _require(type(request["schema_version"]) is int and request["schema_version"] == 1, + "Unsupported request version.", "UnsupportedSchema") + protocol = canonical_decode(self._frozen) + _require(request["task"] == protocol["task"], "Unsupported task revision.", + "UnsupportedRuleVersion") + _require(request["operation"] == protocol["operation"] + and request["validation_scope"] == protocol["validation_scope"], + "Unsupported operation or scope.", "UnsupportedOperation") + _parameters(request["task_parameters"]) + _require(request["comparison_conditions"] == {}, + "The first P1 task uses the configured frozen comparison conditions.") + _expectation(request["expected_candidate"], protocol["candidate_contract"]) + protocol["task_parameters"] = deepcopy(request["task_parameters"]) + protocol["budget_policy_refs"] = deepcopy(request["budget_policy_refs"]) + direct = {item["role"]: item for item in protocol["resources"]} + all_roles = {item["role"]: item for item in _closure(protocol)} + _require(isinstance(request["input_pins"], list), "Expected resource pins.") + seen = set() + for pin in request["input_pins"]: + _resource(pin) + role = pin["role"] + _require(role not in seen, "Duplicate input role.") + seen.add(role) + if role in all_roles: + _require(role in direct + and _project_resource(pin) == _project_resource(direct[role]), + "Input pin conflicts with a frozen task resource.", "IdentityMismatch") + direct[role] = deepcopy(pin) + protocol["resources"] = [direct[role] for role in sorted(direct)] + _protocol(protocol) + closure = _closure(protocol) + for resource in closure: + try: + size, digest = _measure(self._resources(resource["role"])) + except (KeyError, OSError): + raise PreparationError("ArtifactUnavailable", "A required resource is unavailable.") + ref = resource["artifact"] + _require((size, digest) == (ref["size_bytes"], ref["sha256"]), + "Resource bytes disagree with their fixed metadata.", "ArtifactIntegrityError") + data = canonical_encode(protocol) + prepared = { + "schema_version": 1, "prepared_id": "", **{key: deepcopy(protocol[key]) + for key in _COPIES}, + "protocol": {"location": "benchmark/protocol.json", "sha256": sha256(data).hexdigest(), + "size_bytes": len(data), "media_type": "application/json"}, + "protocol_digest": protocol_digest(protocol), + "recipe": deepcopy(protocol["worker_recipe"]), + "expected_candidate": deepcopy(request["expected_candidate"]), + "resources": deepcopy(closure), "created_at": None, "display_notes": "", + } + prepared["prepared_id"] = prepared_id(prepared) + validate_prepared(prepared, data) + roles = frozenset(resource["role"] for resource in closure) + + def read_resource(role): + if role not in roles: + raise KeyError(role) + return self._resources(role) + + return prepared, data, read_resource diff --git a/tests/test_benchmark_preparation.py b/tests/test_benchmark_preparation.py new file mode 100644 index 000000000..39a8ebabb --- /dev/null +++ b/tests/test_benchmark_preparation.py @@ -0,0 +1,334 @@ +"""Production preparation, fixed resource closure and the real Apps T36 consumer.""" + +import json +import os +import subprocess +import sys +from copy import deepcopy +from hashlib import sha256 +from io import StringIO +from pathlib import Path + +import pytest + +from inferrsi.apps.api import prepare as apps_prepare +from inferrsi.apps.handoff.bridge import verify_protocol +from inferrsi.apps.storage.archive import read_archive_artifact, read_prepared +from inferrsi.benchmark import preparation as benchmark +from inferrsi.contracts.json_v1 import CanonicalError, decode, encode + +ROOT = Path(__file__).resolve().parents[1] +GOLDEN = ROOT / "docs/contracts/fixtures/t108-v4" + + +def test_frozen_documentary_ids_and_pure_codec_remain_compatible(monkeypatch): + prepared_bytes = (GOLDEN / "prepared.json").read_bytes() + protocol_bytes = (GOLDEN / "protocol.json").read_bytes() + prepared, protocol = decode(prepared_bytes), decode(protocol_bytes) + monkeypatch.setattr(Path, "open", lambda *a, **k: pytest.fail("codec opened a file")) + assert benchmark.prepared_id(prepared) == prepared["prepared_id"] + assert benchmark.protocol_digest(protocol) == prepared["protocol_digest"] + assert benchmark.encode(prepared) == prepared_bytes + assert benchmark.decode(prepared_bytes) == prepared + benchmark.validate_prepared(prepared, protocol_bytes) + + +@pytest.fixture +def preparation(tmp_path): + protocol = decode((GOLDEN / "protocol.json").read_bytes()) + # Historical vectors stay unchanged. This configured task binds actual code + # bytes through the production resource capability instead of a dummy digest. + source = tmp_path / "worker.py" + source.write_bytes(b"import sys\nprint(sys.version_info.major)\n") + data = tmp_path / "input.txt" + data.write_bytes(b"fixed input\n") + sources = {"worker-code": source, "input-data": data} + pins = [] + for role, path in sorted(sources.items()): + payload = path.read_bytes() + pins.append({ + "role": role, + "artifact": {"location": "source/" + role, "sha256": sha256(payload).hexdigest(), + "size_bytes": len(payload), "media_type": "text/plain"}, + "format_schema": {"name": "utf8-text", "revision": "1"}, + "required_for": ["prepare", "execute"], + }) + protocol["resources"] = pins + protocol["worker_recipe"]["code_sha256"] = sha256(source.read_bytes()).hexdigest() + opened = [] + + def resources(role): + stream = sources[role].open("rb") + opened.append(stream) + return stream + + service = benchmark.PreparationService( + protocol, resources=resources, worker_role="worker-code", + ) + request = { + "schema_version": 1, "task": protocol["task"], "operation": "evaluate", + "validation_scope": "p1", "input_pins": [], + "task_parameters": deepcopy(protocol["task_parameters"]), + "comparison_conditions": {}, "expected_candidate": None, "budget_policy_refs": [], + } + archive = tmp_path / "archive" + archive.mkdir() + config = { + "schema_version": 1, "archive_namespace": "apps", "harness_namespace": "test", + "mounts": [{"prefix": "archives/local", "namespace": "apps", + "root": str(archive), "kind": "directory"}, + {"prefix": "stores/test", "namespace": "test", + "root": str(tmp_path / "store"), "kind": "harness_sqlite"}], + } + return service, request, sources, protocol, opened, config, archive + + +def test_real_prepare_freezes_exact_supplied_bytes_and_isolates_caller_mutations(preparation): + service, request, sources, protocol, opened, _, _ = preparation + protocol["task_parameters"]["label"] = "changed after configuration" + prepared, data, reader = service.prepare(request) + assert all(stream.closed for stream in opened) + benchmark.validate_prepared(prepared, data) + verify_protocol(prepared, data) + assert decode(data)["task_parameters"] == request["task_parameters"] + assert prepared["recipe"]["code_sha256"] == sha256( + sources["worker-code"].read_bytes() + ).hexdigest() + with reader("input-data") as stream: + assert stream.read() == b"fixed input\n" + with pytest.raises(KeyError): + reader("not-in-closure") + original_id = prepared["prepared_id"] + prepared["recipe"]["args"].append("caller mutation") + assert service.prepare(request)[0]["prepared_id"] == original_id + + +def test_two_layer_locations_and_ref_shaped_parameters(preparation): + service, request, _, _, _, _, _ = preparation + prepared, data, _ = service.prepare(request) + moved = deepcopy(prepared) + moved["protocol"]["location"] = "relocated/protocol.json" + for resource in moved["resources"]: + resource["artifact"]["location"] = "relocated/" + resource["role"] + benchmark.validate_prepared(moved, data) + assert benchmark.prepared_id(moved) == prepared["prepared_id"] + assert benchmark.encode(moved) != benchmark.encode(prepared) + + internal = decode(data) + internal["resources"][0]["artifact"]["location"] = "inside/new-location" + internal_bytes = encode(internal) + assert internal_bytes != data + assert benchmark.protocol_digest(internal) == prepared["protocol_digest"] + moved["protocol"].update(sha256=sha256(internal_bytes).hexdigest(), + size_bytes=len(internal_bytes)) + moved["prepared_id"] = benchmark.prepared_id(moved) + assert moved["prepared_id"] != prepared["prepared_id"] + benchmark.validate_prepared(moved, internal_bytes) + + # Identity projection does not recursively strip a location from Parameters. + projected = decode(data) + projected["task_parameters"]["ref_shaped"] = deepcopy(prepared["protocol"]) + first = benchmark.protocol_digest(projected) + projected["task_parameters"]["ref_shaped"]["location"] = "semantic/location" + assert benchmark.protocol_digest(projected) != first + changed = decode(data) + changed["task_parameters"]["label"] = "different frozen condition" + assert benchmark.protocol_digest(changed) != prepared["protocol_digest"] + request["task_parameters"]["label"] = changed["task_parameters"]["label"] + new_prepared, new_data, _ = service.prepare(request) + assert new_data != data + for key in ("prepared_id", "protocol_digest"): + assert new_prepared[key] != prepared[key] + assert new_prepared["protocol"]["sha256"] != prepared["protocol"]["sha256"] + + +@pytest.mark.parametrize("change,code", [ + (lambda value: value.update(schema_version=2), "UnsupportedSchema"), + (lambda value: value["task"].update(revision="unsupported"), "UnsupportedRuleVersion"), + (lambda value: value.update(operation="other"), "UnsupportedOperation"), + (lambda value: value.update(unrecognized=True), "InvalidRequest"), + (lambda value: value["task_parameters"].update(decimal_fixture="1.0"), "InvalidRequest"), + (lambda value: value.update(expected_candidate={"candidate_id": "incomplete"}), + "InvalidRequest"), + (lambda value: value.update(expected_candidate={ + "candidate_id": "candidate", "sha256": "a" * 64, + "format_schema": {"name": "other", "revision": "1"}, + }), "InvalidRequest"), +]) +def test_request_rejections_are_wire_errors_before_any_archive(preparation, change, code): + service, request, _, _, opened, _, archive = preparation + change(request) + response = service.prepare(request) + assert response["code"] == code and response["stage"] == "prepare" + assert response["latest_record_ref"] is None and response["retained_refs"] == [] + assert list(archive.iterdir()) == [] and opened == [] + + +@pytest.mark.parametrize("stage", ["prepare", "archive"]) +def test_t36_resource_loss_at_each_boundary(preparation, tmp_path, stage): + service, request, sources, _, _, config, archive = preparation + if stage == "prepare": + sources["input-data"].unlink() + producer = service.prepare + else: + response = service.prepare(request) + sources["input-data"].unlink() + def producer(value): + return response + result = apps_prepare( + {"schema_version": 1, "request": request}, workspace=tmp_path, + storage_config=config, prepare_benchmark=producer, + encode_prepared=benchmark.encode, validate_prepared=benchmark.validate_prepared, + ) + assert result.result["reason_code"] == "ArtifactUnavailable", result.result + assert result.result["error"]["stage"] == ("prepare" if stage == "prepare" + else "prepared_archive") + assert result.result["prepared_ref"] is None + assert list(archive.iterdir()) == [] + + +def test_wrong_resource_bytes_are_not_frozen_as_a_different_pin(preparation): + service, request, sources, _, _, _, _ = preparation + sources["worker-code"].write_bytes(b"unexpected replacement worker") + assert service.prepare(request)["code"] == "ArtifactIntegrityError" + + +@pytest.mark.parametrize("change", [ + lambda value: value["candidate_contract"].update(unknown=True), + lambda value: value["recipe"].update(unrecognized=True), + lambda value: value["recipe"]["outputs"][0].update(required=1), + lambda value: value.update(created_at="2026-02-30T00:00:00Z"), + lambda value: value.update(created_at=True), + lambda value: value.update(created_at="2026-11-20T00:00:00.1234567890Z"), + lambda value: value["resources"].append(deepcopy(value["resources"][0])), + lambda value: value["required_evidence"][0].update(matching_rule_revision="other"), +]) +def test_closed_prepared_shape_rejected_even_with_recomputed_identity(preparation, change): + service, request, _, _, _, _, _ = preparation + prepared, _, _ = service.prepare(request) + change(prepared) + prepared["prepared_id"] = benchmark.prepared_id(prepared) + with pytest.raises(ValueError): + benchmark.encode(prepared) + + +def test_unsafe_location_is_rejected_by_the_encoder_itself(preparation): + service, request, _, _, _, _, _ = preparation + prepared, _, _ = service.prepare(request) + prepared["protocol"]["location"] = "../escape" + with pytest.raises(benchmark.PreparationError, match="logical artifact location"): + benchmark.encode(prepared) + + +def test_source_change_after_preparation_is_rejected_by_independent_apps_verification( + preparation, tmp_path, +): + service, request, sources, _, _, config, archive = preparation + response = service.prepare(request) + sources["input-data"].write_bytes(b"changed input") + result = apps_prepare( + {"schema_version": 1, "request": request}, workspace=tmp_path, + storage_config=config, prepare_benchmark=lambda value: response, + encode_prepared=benchmark.encode, validate_prepared=benchmark.validate_prepared, + ) + assert result.result["reason_code"] == "ArtifactIntegrityError" + assert result.result["prepared_ref"] is None and list(archive.iterdir()) == [] + + +def test_frozen_task_rules_are_not_request_parameters(preparation): + _, _, sources, protocol, _, _, _ = preparation + protocol["claim_definitions"][0]["decision_rule"]["revision"] = "unimplemented" + with pytest.raises(benchmark.PreparationError) as error: + benchmark.PreparationService( + protocol, worker_role="worker-code", resources=lambda role: sources[role].open("rb"), + ) + assert error.value.code == "UnsupportedRuleVersion" + + +@pytest.mark.parametrize("failure", [ValueError("host bug"), CanonicalError("host", "host bug")]) +def test_programming_error_in_a_host_capability_is_not_a_port_rejection(preparation, failure): + _, request, _, protocol, _, _, _ = preparation + + def broken(role): + raise failure + + service = benchmark.PreparationService(protocol, resources=broken, worker_role="worker-code") + with pytest.raises(ValueError) as error: + service.prepare(request) + assert error.value is failure + + +def test_text_stream_is_a_closed_host_capability_error(preparation): + _, request, _, protocol, _, _, _ = preparation + stream = StringIO("a text capability is not a binary resource") + service = benchmark.PreparationService( + protocol, resources=lambda role: stream, worker_role="worker-code", + ) + with pytest.raises(TypeError, match="binary bytes"): + service.prepare(request) + assert stream.closed + + +def test_provisional_locations_do_not_replace_role_authority_or_archive_paths( + preparation, tmp_path, +): + service, request, sources, protocol, _, config, _ = preparation + pin = deepcopy(protocol["resources"][0]) + assert pin["role"] == "input-data" + pin["artifact"]["location"] = protocol["resources"][1]["artifact"]["location"] + request["input_pins"] = [pin] + result = apps_prepare( + {"schema_version": 1, "request": request}, workspace=tmp_path, + storage_config=config, prepare_benchmark=service.prepare, + encode_prepared=benchmark.encode, validate_prepared=benchmark.validate_prepared, + ) + assert result.result["exit_code"] == 0, result.result + prepared = read_prepared(config, result.result["prepared_ref"]) + assert len({item["artifact"]["location"] for item in prepared["resources"]}) == 2 + assert {item["role"]: read_archive_artifact(config, item["artifact"]) + for item in prepared["resources"]} == { + role: path.read_bytes() for role, path in sources.items() + } + frozen = decode(read_archive_artifact(config, prepared["protocol"])) + assert len({item["artifact"]["location"] for item in frozen["resources"]}) == 1 + + +def test_t36_real_port_archive_and_cross_process_fixed_readback(preparation, tmp_path): + service, request, sources, _, _, config, _ = preparation + expected, protocol, _ = service.prepare(request) + result = apps_prepare( + {"schema_version": 1, "request": request}, workspace=tmp_path, + storage_config=config, prepare_benchmark=service.prepare, + encode_prepared=benchmark.encode, validate_prepared=benchmark.validate_prepared, + ) + assert result.result["exit_code"] == 0, result.result + ref = result.result["prepared_ref"] + assert ref["prepared_id"] == expected["prepared_id"] + archived = read_prepared(config, ref) + benchmark.validate_prepared(archived, protocol) + child = """ +import json, sys +sys.path.insert(0, sys.argv[1]) +from inferrsi.apps.storage.archive import read_prepared, read_archive_artifact +from inferrsi.benchmark.preparation import encode, validate_prepared +config, ref = json.loads(sys.stdin.read()) +prepared = read_prepared(config, ref) +protocol = read_archive_artifact(config, prepared['protocol']) +validate_prepared(prepared, protocol) +print(json.dumps({'prepared': encode(prepared).hex(), 'protocol': protocol.hex(), + 'resources': {x['role']: read_archive_artifact(config, x['artifact']).hex() + for x in prepared['resources']}})) +""" + process = subprocess.run( + [sys.executable, "-I", "-c", child, str(ROOT / "src")], cwd=tmp_path, + env={**os.environ, "PYTHONUTF8": "1"}, input=json.dumps([config, ref]), + capture_output=True, text=True, timeout=30, + ) + assert process.returncode == 0, process.stderr + observed = json.loads(process.stdout) + prepared_bytes = bytes.fromhex(observed["prepared"]) + assert sha256(prepared_bytes).hexdigest() == ref["artifact"]["sha256"] + assert len(prepared_bytes) == ref["artifact"]["size_bytes"] + assert observed["protocol"] == protocol.hex() + assert observed["resources"] == {role: path.read_bytes().hex() + for role, path in sources.items()}