From 8980ff6ed205744998923877ad34d946eba368db Mon Sep 17 00:00:00 2001 From: zengchuang Date: Fri, 20 Nov 2026 09:28:23 +0800 Subject: [PATCH 1/5] feat(benchmark): freeze configured P1 preparations Agent-assisted-by: Codex (author) Signed-off-by: zengchuang --- docs/contracts/production-prepare.md | 92 ++++++ docs/operations.md | 1 + src/inferrsi/benchmark/preparation.py | 444 ++++++++++++++++++++++++++ tests/test_benchmark_preparation.py | 285 +++++++++++++++++ 4 files changed, 822 insertions(+) create mode 100644 docs/contracts/production-prepare.md create mode 100644 src/inferrsi/benchmark/preparation.py create mode 100644 tests/test_benchmark_preparation.py diff --git a/docs/contracts/production-prepare.md b/docs/contracts/production-prepare.md new file mode 100644 index 000000000..0b88a1495 --- /dev/null +++ b/docs/contracts/production-prepare.md @@ -0,0 +1,92 @@ +# Production P1 preparation + +`inferrsi.benchmark.preparation.PreparationService.prepare` implements the +Benchmark preparation port for the accepted D023 / `t108-v4` task family: +`p1-fixture@1`, `p1-rules@1`, operation `evaluate`, scope `p1`. + +The trusted host configures a **Benchmark-owned frozen ProtocolV1** and a +read-only resource capability. Neither is supplied inside PrepareRequest. +The task configuration includes an explicit resource role for the actual worker +code bytes. Its ArtifactRef digest must equal `worker_recipe.code_sha256`. +Preparation verifies the exact supplied bytes; it never treats the historical +documentation worker digest as an available executable. + +```python +from inferrsi.benchmark.preparation import ( + PreparationService, encode, decode, validate_prepared, +) + +service = PreparationService( + frozen_protocol, resources=open_fixed_resource, worker_role="worker-code", +) +prepared, protocol_bytes, open_resource = service.prepare(request) +``` + +`open_fixed_resource(role)` is a host-supplied capability returning bytes or a +fresh binary stream for an authorized role. The caller owns closing each returned +stream. It does not interpret a request-provided location as a filesystem path. +The frozen resource metadata is verified during preparation. The returned +`open_resource` accepts only roles in that preparation's closure and reopens the +resource through the same capability. It does not write or archive bytes. + +The first task keeps the P1 candidate, claims, evidence rules, worker recipe +shape and sampling conditions. PrepareRequest can bind its candidate expectation, +P1 task parameters, budget references and additional resource pins. A supplied +pin cannot replace a frozen role's hash, format or required use. The first task +accepts an empty `comparison_conditions`: comparison conditions are configured +in the frozen task definition, rather than silently accepting unknown overrides. +Unknown task, grader, worker or claim-rule revisions are refused. + +The protocol contains its declared resources and optional baseline/reference/ +quality-reference resources. PreparedEvaluation carries the complete closure, +sorted by unique role. Duplicate roles across any of these positions are refused. + +## Identity and pure codecs + +`encode(prepared)`, `decode(bytes)` and `validate_prepared(prepared, +protocol_bytes)` perform no artifact reads, worker imports, execution or writes. +They preserve the canonical `inferrsi-json-v1` byte contract. + +- `prepared_id` excludes its own value, `created_at`, `display_notes`, and + locations in its schema-declared outer ArtifactRefs. Moving both the outer + protocol and outer resource references preserves this ID. +- `protocol_digest` excludes protocol display notes and locations only in the + declared resource references, including baseline/reference/quality-reference. + It preserves an arbitrary ref-shaped value inside Parameters. +- Moving a reference **inside** the protocol changes the protocol's canonical + bytes and byte digest. The changed digest changes `prepared_id`, even when + `protocol_digest` remains equal. Other semantic changes affect both identities. + +## Apps boundary and failures + +Inject the bound port and pure functions into the existing Apps API: + +```python +from inferrsi.apps.api import prepare + +output = prepare( + {"schema_version": 1, "request": request}, + workspace=workspace, storage_config=storage_config, + prepare_benchmark=service.prepare, + encode_prepared=encode, validate_prepared=validate_prepared, +) +``` + +Apps assigns its final outer locations, invokes the Benchmark pure codecs, +verifies bytes independently, then commits the full closure before publishing +a PreparedRef. Benchmark imports neither Apps nor Harness. + +The prepare port returns a portable PublicError wire object for a declared +rejection. Missing resources produce `ArtifactUnavailable` at `prepare`; +incorrect retained metadata produces `ArtifactIntegrityError`. No preparation +is archived by Benchmark. If the resource disappears after preparation, the +Apps archive reports `ArtifactUnavailable` at `prepared_archive`; changed bytes +are rejected by its independent integrity verifier. Neither case publishes a +PreparedRef or a partial prepared closure. + +This port does not register the configured worker with a Harness target or +execute/assess it. The H evaluation-mode path and production offline assessor +remain separate deliveries under #429 and #434. The packaged engineering-task +registry is unchanged. A host must supply the reviewed P1 task definition and +actual worker/resource capability; no default fixture backend is silently +selected. Ordinary CPU regressions do not establish a model/service gain. diff --git a/docs/operations.md b/docs/operations.md index 1e1842786..db301c7ae 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -51,6 +51,7 @@ Effect classes used below: | Rebuild scoped legacy evidence views | `python tools/rebuild_model_evidence.py --archive --output ` | CPU-only; creates a new output directory | Authenticates original bytes and pinned projection/code; no extraction, network, archived-code execution or campaign qualification | | Inspect evidence inventory / generate budget input | `python tools/validate_evidence.py [--budget-assets]` | Read-only | Offline layout/hash/schema checks; optional disposable #328 metadata view derived from campaign authority; no downloads | | Execute one ordinary local CPU command with a fixed record | Python `inferrsi.harness.api.Harness.allocate` / `execute` / `read` | Experiment-executing / record-writing | One explicit RunStore root and one trusted local CPU target; ordinary command mode only; a record holds execution facts, so exit 0 is not a verdict; no device, model or network fallback | +| Freeze a configured P1 preparation | Python `benchmark.preparation.PreparationService.prepare`; [production prepare](contracts/production-prepare.md) | Read-only resource access / pure encoding | Verifies supplied frozen worker/resource bytes and returns a preparation, protocol and role-keyed closure; Apps separately archives them before publishing a Ref | | Visualize a recorded run or campaign | CLI `dashboard`; [dashboard](dashboard.md) | Report-generating / local read-only serving | Writes an offline HTML snapshot or serves an optional live view on loopback; does not run an agent or validate a claim | | Capture a knowledge context | CLI `campaign capture-context`; [offline dashboard](dashboard.md#record-the-optimization-story-campaign-trace) | Local evidence-writing | Saves assembled context and resource hashes in a new `knowledge/.json`. Supports ordinary questions and explicit IDs. The only accepted explicit `--relevance-method` value is `find-lexical-v1` | | Retain an executed retrieval | Python `apps.workflows.campaign.save_retrieval`; [capture contract](knowledge-retrieval-trace.md#retain-a-standalone-result) | Local evidence-writing | Saves an existing catalog/collect result under a fresh campaign capture ID; no new search or automatic publication | diff --git a/src/inferrsi/benchmark/preparation.py b/src/inferrsi/benchmark/preparation.py new file mode 100644 index 000000000..7b187e975 --- /dev/null +++ b/src/inferrsi/benchmark/preparation.py @@ -0,0 +1,444 @@ +"""Frozen D023/P1 preparations over a Benchmark-owned task and byte capability. + +The host configures the frozen task and a role-keyed read-only resource opener. +PrepareRequest cannot supply task rules, worker code or a filesystem authority. +Preparation neither executes the worker nor persists a preparation or its bytes. +""" + +from __future__ import annotations + +import re +from collections.abc import Callable +from contextlib import closing +from copy import deepcopy +from datetime import datetime +from hashlib import sha256 +from pathlib import PurePosixPath + +from ..contracts.json_v1 import CanonicalError +from ..contracts.json_v1 import decode as canonical_decode +from ..contracts.json_v1 import encode as canonical_encode + +_REQUEST = set("schema_version task operation validation_scope input_pins task_parameters " + "comparison_conditions expected_candidate budget_policy_refs".split()) +_PROTOCOL = set("schema_version task grader operation validation_scope candidate_contract " + "worker_recipe input_requirements claim_definitions metric_definitions " + "required_evidence task_parameters baseline reference quality_reference " + "sampling feedback_visibility budget_policy_refs resources display_notes".split()) +_PREPARED = set("schema_version prepared_id task grader operation validation_scope protocol " + "protocol_digest candidate_contract expected_candidate required_evidence " + "recipe resources created_at display_notes".split()) +_COPIES = ("task", "grader", "operation", "validation_scope", "candidate_contract", + "required_evidence") +_RESOURCE = {"role", "artifact", "format_schema", "required_for"} +_ARTIFACT = {"location", "sha256", "size_bytes", "media_type"} +_ID = re.compile(r"[A-Za-z0-9][A-Za-z0-9._:/@+\-]{0,255}\Z") +_SHA = re.compile(r"[a-f0-9]{64}\Z") +_DECIMAL = re.compile(r"(?:0|-?[1-9][0-9]*|-?(?:0|[1-9][0-9]*)\.[0-9]*[1-9])\Z") +_CANDIDATE = { + "kind": "text", "input_role": "candidate", + "format_schema": {"name": "utf8-text", "revision": "1"}, + "identity_rule": {"name": "captured-sha256", "revision": "1"}, + "entrypoint": None, "required_paths": [], "allowed_content": {}, + "modification_receipt_required": False, +} + + +class PreparationError(ValueError): + """A Benchmark-owned rejection, with the portable PublicError wire form.""" + + def __init__(self, code: str, message: str): + super().__init__(message) + self.code = code + + def to_json(self) -> dict: + return { + "code": self.code, "stage": "prepare", "attempt_id": None, "started": None, + "latest_record_ref": None, "partial_record": None, "retained_refs": [], + "field_errors": [], "message": str(self), + } + + +def _require(condition, message: str, code: str = "InvalidRequest") -> None: + if not condition: + raise PreparationError(code, message) + + +def _object(value, fields) -> None: + _require(isinstance(value, dict) and value.keys() == fields, "Unexpected object fields.") + + +def _id(value) -> bool: + return isinstance(value, str) and _ID.fullmatch(value) is not None + + +def _definition(value) -> None: + _object(value, {"name", "revision"}) + _require(all(_id(item) for item in value.values()), "Invalid definition reference.") + + +def _artifact(value) -> None: + _object(value, _ARTIFACT) + location = value["location"] + _require(isinstance(location, str) and bool(location) and "\\" not in location + and ":" not in location and not location.startswith("/") + and all(part not in ("", ".", "..") for part in location.split("/")) + and str(PurePosixPath(location)) == location, "Invalid logical artifact location.") + _require(isinstance(value["sha256"], str) and _SHA.fullmatch(value["sha256"]) is not None + and type(value["size_bytes"]) is int and value["size_bytes"] >= 0 + and isinstance(value["media_type"], str) and bool(value["media_type"]), + "Invalid artifact metadata.") + + +def _resource(value) -> None: + _object(value, _RESOURCE) + _require(_id(value["role"]), "Invalid resource role.") + _artifact(value["artifact"]) + _definition(value["format_schema"]) + uses = value["required_for"] + _require(isinstance(uses, list) and bool(uses) + and all(isinstance(use, str) for use in uses) + and len(uses) == len(set(uses)) and set(uses) <= {"prepare", "execute", "assess"}, + "Invalid resource use.") + + +def _closure(protocol: dict) -> list[dict]: + _require(isinstance(protocol["resources"], list), "Expected protocol resources.") + values = list(protocol["resources"]) + values += [protocol[key] for key in ("baseline", "reference", "quality_reference") + if protocol[key] is not None] + for value in values: + _resource(value) + roles = [value["role"] for value in values] + _require(len(roles) == len(set(roles)), "Resource roles must be unique across the closure.") + return sorted(values, key=lambda value: value["role"]) + + +def _project_resource(value: dict) -> dict: + _resource(value) + return {**value, "artifact": {key: item for key, item in value["artifact"].items() + if key != "location"}} + + +def protocol_digest(protocol: dict) -> str: + """Project declared ArtifactRefs only; ref-shaped Parameters stay semantic.""" + _object(protocol, _PROTOCOL) + value = deepcopy(protocol) + value.pop("display_notes") + value["resources"] = [_project_resource(item) for item in value["resources"]] + for key in ("baseline", "reference", "quality_reference"): + if value[key] is not None: + value[key] = _project_resource(value[key]) + return "sha256:" + sha256(canonical_encode(value)).hexdigest() + + +def prepared_id(prepared: dict) -> str: + """Compute identity independently of outer locators, clock and display notes.""" + _object(prepared, _PREPARED) + value = deepcopy(prepared) + for key in ("prepared_id", "created_at", "display_notes"): + value.pop(key) + _artifact(value["protocol"]) + value["protocol"].pop("location") + value["resources"] = [_project_resource(item) for item in value["resources"]] + return "sha256:" + sha256(canonical_encode(value)).hexdigest() + + +def _parameters(value) -> None: + _object(value, {"decimal_fixture", "label"}) + _require(isinstance(value["decimal_fixture"], str) + and _DECIMAL.fullmatch(value["decimal_fixture"]) is not None + and isinstance(value["label"], str), "Invalid P1 task parameters.") + + +def _candidate(value) -> None: + _require(canonical_encode(value) == canonical_encode(_CANDIDATE), + "Unsupported P1 candidate contract.", "UnsupportedRuleVersion") + + +def _recipe(recipe) -> None: + _object(recipe, set("recipe_revision runtime_role entrypoint_kind entrypoint " + "entrypoint_revision code_sha256 args input_roles outputs " + "required_capabilities".split())) + fixed = {"recipe_revision": "1", "runtime_role": "local-python", + "entrypoint_kind": "python_module", "entrypoint": "fixture_worker", + "entrypoint_revision": "1"} + _require(all(recipe[key] == value for key, value in fixed.items()), + "Unsupported frozen P1 worker revision.", "UnsupportedRuleVersion") + _require(isinstance(recipe["code_sha256"], str) + and _SHA.fullmatch(recipe["code_sha256"]) is not None, + "Invalid frozen worker digest.") + for key in ("args", "input_roles", "required_capabilities"): + _require(isinstance(recipe[key], list) + and all(isinstance(item, str) and item for item in recipe[key]), + "Invalid worker arguments or roles.") + _require(recipe["input_roles"] == ["candidate"] + and recipe["args"] == ["{input:candidate}", "{output:grader}"] + and recipe["required_capabilities"] == ["local-process-tree"], + "Unsupported P1 worker recipe.", "UnsupportedRuleVersion") + _require(recipe["outputs"] == [{ + "role": "grader", "path": "outputs/grader.json", "media_type": "application/json", + "format_schema": {"name": "fixture-grader", "revision": "1"}, "required": True, + }] and type(recipe["outputs"][0]["required"]) is bool, + "Unsupported P1 worker output.", "UnsupportedRuleVersion") + + +def _requirements(requirements) -> None: + _require(isinstance(requirements, list) and len(requirements) == 4, + "Expected the P1 evidence requirements.") + expected = { + "format": ("F", "preparing", {"harness_capture"}), + "grader": ("C", "finalizing", {"task_worker"}), + "samples": ("M", "running", {"harness_capture", "task_worker"}), + "cleanup": ("M", "finalizing", {"harness_capture"}), + } + for item in requirements: + _object(item, set("check_id checker_revision allowed_producer_kinds subject phase " + "artifact_schema matching_rule_revision allowed_control_domains " + "for_claims".split())) + key = item["check_id"] + _require(isinstance(key, str) and key in expected, "Unsupported P1 check.", + "UnsupportedRuleVersion") + claim, phase, producers = expected.pop(key) + kinds, domains = item["allowed_producer_kinds"], item["allowed_control_domains"] + _require(isinstance(kinds, list) and all(_id(kind) for kind in kinds) + and len(kinds) == len(set(kinds)) and set(kinds) == producers + and isinstance(domains, list) and bool(domains) + and all(_id(domain) for domain in domains) + and len(domains) == len(set(domains)) + and item["checker_revision"] == item["matching_rule_revision"] == "1" + and item["subject"] == "candidate" and item["phase"] == phase + and item["for_claims"] == [claim] + and item["artifact_schema"] == {"name": key + "-proof", "revision": "1"}, + "Unsupported frozen P1 evidence rule.", "UnsupportedRuleVersion") + + +def _protocol(protocol: dict) -> None: + canonical_encode(protocol) + _object(protocol, _PROTOCOL) + _require(type(protocol["schema_version"]) is int and protocol["schema_version"] == 1, + "Unsupported protocol version.", "UnsupportedSchema") + _require(protocol["task"] == {"name": "p1-fixture", "revision": "1"} + and protocol["grader"] == {"name": "p1-rules", "revision": "1"}, + "Only the accepted P1 task and grader revision are supported.", + "UnsupportedRuleVersion") + _require(protocol["operation"] == "evaluate" and protocol["validation_scope"] == "p1", + "Unsupported task operation or scope.", "UnsupportedOperation") + _parameters(protocol["task_parameters"]) + _closure(protocol) + _candidate(protocol["candidate_contract"]) + _recipe(protocol["worker_recipe"]) + _requirements(protocol["required_evidence"]) + _require(protocol["input_requirements"] == {"candidate_role": "candidate"} + and protocol["metric_definitions"] == [] + and protocol["sampling"] == {"order": ["AB", "BA"], "repetitions": 2} + and type(protocol["sampling"]["repetitions"]) is int + and protocol["feedback_visibility"] == ["format"] + and isinstance(protocol["display_notes"], str), + "Unsupported P1 protocol conditions.", "UnsupportedRuleVersion") + claims = protocol["claim_definitions"] + _require(isinstance(claims, list) and len(claims) == 3, "Expected the P1 claim registry.") + expected = {"F": ["format"], "C": ["grader"], "M": ["samples", "cleanup"]} + for claim in claims: + _object(claim, {"claim_id", "validation_scope", "decision_rule", "required_checks", + "requires_completed_measurement", "requires_cleanup_confirmed"}) + key = claim["claim_id"] + _require(isinstance(key, str) and key in expected + and claim["decision_rule"] == {"name": "p1-" + key, "revision": "1"} + and claim["validation_scope"] == "p1" + and claim["required_checks"] == expected.pop(key) + and type(claim["requires_completed_measurement"]) is bool + and type(claim["requires_cleanup_confirmed"]) is bool + and claim["requires_completed_measurement"] == (key == "M") + and claim["requires_cleanup_confirmed"] == (key == "M"), + "Unsupported frozen P1 claim rule.", "UnsupportedRuleVersion") + _require(isinstance(protocol["budget_policy_refs"], list), "Expected budget references.") + for ref in protocol["budget_policy_refs"]: + _definition(ref) + + +def _prepared(prepared: dict) -> None: + canonical_encode(prepared) + _object(prepared, _PREPARED) + _require(type(prepared["schema_version"]) is int and prepared["schema_version"] == 1, + "Unsupported prepared version.", "UnsupportedSchema") + _artifact(prepared["protocol"]) + _candidate(prepared["candidate_contract"]) + _recipe(prepared["recipe"]) + _requirements(prepared["required_evidence"]) + _require(prepared["task"] == {"name": "p1-fixture", "revision": "1"} + and prepared["grader"] == {"name": "p1-rules", "revision": "1"} + and prepared["operation"] == "evaluate" and prepared["validation_scope"] == "p1", + "Unsupported prepared task revision or scope.", "UnsupportedRuleVersion") + _require(isinstance(prepared["display_notes"], str) + and isinstance(prepared["protocol_digest"], str) + and re.fullmatch(r"sha256:[a-f0-9]{64}", prepared["protocol_digest"]) is not None, + "Invalid preparation metadata.") + timestamp = prepared["created_at"] + if timestamp is not None: + _require(isinstance(timestamp, str) + and re.fullmatch(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,9})?Z", + timestamp) is not None, + "Invalid preparation timestamp.") + try: + datetime.fromisoformat(timestamp.replace("Z", "+00:00")) + except ValueError: + raise PreparationError("InvalidRequest", "Invalid preparation timestamp.") from None + _require(isinstance(prepared["resources"], list), "Expected prepared resources.") + for resource in prepared["resources"]: + _resource(resource) + roles = [resource["role"] for resource in prepared["resources"]] + _require(roles == sorted(set(roles)), "Prepared closure must be sorted by unique role.") + candidate = prepared["expected_candidate"] + if candidate is not None: + _object(candidate, {"candidate_id", "sha256", "format_schema"}) + _require(_id(candidate["candidate_id"]) and isinstance(candidate["sha256"], str) + and _SHA.fullmatch(candidate["sha256"]) is not None, + "Invalid expected candidate.") + _definition(candidate["format_schema"]) + _require(candidate["format_schema"] == prepared["candidate_contract"]["format_schema"], + "Expected candidate format differs from the frozen contract.") + _require(prepared["prepared_id"] == prepared_id(prepared), + "Prepared identity does not match its content.", "IdentityMismatch") + + +def encode(prepared: dict) -> bytes: + """Pure prepared encoding, including its self-identity check.""" + _prepared(prepared) + return canonical_encode(prepared) + + +def decode(data: bytes) -> dict: + """Decode canonical prepared bytes without resolving or opening any artifact.""" + value = canonical_decode(data) + _require(canonical_encode(value) == data, "Prepared bytes are not canonical.") + _prepared(value) + return value + + +def validate_prepared(prepared: dict, protocol_bytes: bytes) -> None: + """Pure validation of the full prepared/protocol identity and closure binding.""" + _prepared(prepared) + protocol = canonical_decode(protocol_bytes) + _require(canonical_encode(protocol) == protocol_bytes, "Protocol bytes are not canonical.") + _protocol(protocol) + ref = prepared["protocol"] + _require(ref["sha256"] == sha256(protocol_bytes).hexdigest() + and ref["size_bytes"] == len(protocol_bytes) + and ref["media_type"] == "application/json" + and prepared["protocol_digest"] == protocol_digest(protocol) + and all(prepared[key] == protocol[key] for key in _COPIES) + and prepared["recipe"] == protocol["worker_recipe"] + and [_project_resource(item) for item in prepared["resources"]] + == [_project_resource(item) for item in _closure(protocol)], + "Prepared object disagrees with its frozen protocol.", "IdentityMismatch") + + +def _measure(source) -> tuple[int, str]: + if source is None: + raise FileNotFoundError("A required resource is unavailable.") + if isinstance(source, bytes): + return len(source), sha256(source).hexdigest() + size, digest = 0, sha256() + with closing(source): + while True: + chunk = source.read(1024 * 1024) + _require(isinstance(chunk, bytes), "Resource stream must return bytes.") + if not chunk: + break + size += len(chunk) + digest.update(chunk) + return size, digest.hexdigest() + + +class PreparationService: + """The supported P1 prepare port bound to frozen rules and read capabilities. + + ``worker_role`` identifies the retained code bytes for recipe.code_sha256. + Each opener call gives its stream to the caller, which owns closing it. + The returned closure reopens resources: deletion fails at the Apps archive + boundary and changed bytes are independently rejected by Apps' verifier. + """ + + def __init__(self, protocol: dict, *, resources: Callable, worker_role: str): + _protocol(protocol) + _require(callable(resources) and _id(worker_role), "Invalid resource capability.") + closure = {item["role"]: item for item in _closure(protocol)} + _require(worker_role in closure and closure[worker_role]["artifact"]["sha256"] + == protocol["worker_recipe"]["code_sha256"], + "The worker recipe must bind retained code bytes.", "IdentityMismatch") + self._frozen = canonical_encode(protocol) + self._resources = resources + + def prepare(self, request: dict) -> tuple | dict: + """Return the prepared object, frozen protocol bytes and read-only closure.""" + try: + canonical_encode(request) + except CanonicalError: + return PreparationError("InvalidRequest", "Malformed prepare request.").to_json() + try: + return self._prepare(request) + except PreparationError as error: + return error.to_json() + + def _prepare(self, request: dict) -> tuple: + _object(request, _REQUEST) + _require(type(request["schema_version"]) is int and request["schema_version"] == 1, + "Unsupported request version.", "UnsupportedSchema") + protocol = canonical_decode(self._frozen) + _require(request["task"] == protocol["task"], "Unsupported task revision.", + "UnsupportedRuleVersion") + _require(request["operation"] == protocol["operation"] + and request["validation_scope"] == protocol["validation_scope"], + "Unsupported operation or scope.", "UnsupportedOperation") + _parameters(request["task_parameters"]) + _require(request["comparison_conditions"] == {}, + "The first P1 task uses the configured frozen comparison conditions.") + protocol["task_parameters"] = deepcopy(request["task_parameters"]) + protocol["budget_policy_refs"] = deepcopy(request["budget_policy_refs"]) + direct = {item["role"]: item for item in protocol["resources"]} + all_roles = {item["role"]: item for item in _closure(protocol)} + _require(isinstance(request["input_pins"], list), "Expected resource pins.") + seen = set() + for pin in request["input_pins"]: + _resource(pin) + role = pin["role"] + _require(role not in seen, "Duplicate input role.") + seen.add(role) + if role in all_roles: + _require(role in direct + and _project_resource(pin) == _project_resource(direct[role]), + "Input pin conflicts with a frozen task resource.", "IdentityMismatch") + direct[role] = deepcopy(pin) + protocol["resources"] = [direct[role] for role in sorted(direct)] + _protocol(protocol) + closure = _closure(protocol) + for resource in closure: + try: + size, digest = _measure(self._resources(resource["role"])) + except (KeyError, OSError): + raise PreparationError("ArtifactUnavailable", "A required resource is unavailable.") + ref = resource["artifact"] + _require((size, digest) == (ref["size_bytes"], ref["sha256"]), + "Resource bytes disagree with their fixed metadata.", "ArtifactIntegrityError") + data = canonical_encode(protocol) + prepared = { + "schema_version": 1, "prepared_id": "", **{key: deepcopy(protocol[key]) + for key in _COPIES}, + "protocol": {"location": "benchmark/protocol.json", "sha256": sha256(data).hexdigest(), + "size_bytes": len(data), "media_type": "application/json"}, + "protocol_digest": protocol_digest(protocol), + "recipe": deepcopy(protocol["worker_recipe"]), + "expected_candidate": deepcopy(request["expected_candidate"]), + "resources": deepcopy(closure), "created_at": None, "display_notes": "", + } + prepared["prepared_id"] = prepared_id(prepared) + validate_prepared(prepared, data) + roles = frozenset(resource["role"] for resource in closure) + + def read_resource(role): + if role not in roles: + raise KeyError(role) + return self._resources(role) + + return prepared, data, read_resource diff --git a/tests/test_benchmark_preparation.py b/tests/test_benchmark_preparation.py new file mode 100644 index 000000000..d85cfa365 --- /dev/null +++ b/tests/test_benchmark_preparation.py @@ -0,0 +1,285 @@ +"""Production preparation, fixed resource closure and the real Apps T36 consumer.""" + +import json +import os +import subprocess +import sys +from copy import deepcopy +from hashlib import sha256 +from pathlib import Path + +import pytest + +from inferrsi.apps.api import prepare as apps_prepare +from inferrsi.apps.handoff.bridge import verify_protocol +from inferrsi.apps.storage.archive import read_prepared +from inferrsi.benchmark import preparation as benchmark +from inferrsi.contracts.json_v1 import CanonicalError, decode, encode + +ROOT = Path(__file__).resolve().parents[1] +GOLDEN = ROOT / "docs/contracts/fixtures/t108-v4" + + +def test_frozen_documentary_ids_and_pure_codec_remain_compatible(monkeypatch): + prepared_bytes = (GOLDEN / "prepared.json").read_bytes() + protocol_bytes = (GOLDEN / "protocol.json").read_bytes() + prepared, protocol = decode(prepared_bytes), decode(protocol_bytes) + monkeypatch.setattr(Path, "open", lambda *a, **k: pytest.fail("codec opened a file")) + assert benchmark.prepared_id(prepared) == prepared["prepared_id"] + assert benchmark.protocol_digest(protocol) == prepared["protocol_digest"] + assert benchmark.encode(prepared) == prepared_bytes + assert benchmark.decode(prepared_bytes) == prepared + benchmark.validate_prepared(prepared, protocol_bytes) + + +@pytest.fixture +def preparation(tmp_path): + protocol = decode((GOLDEN / "protocol.json").read_bytes()) + # Historical vectors stay unchanged. This configured task binds actual code + # bytes through the production resource capability instead of a dummy digest. + source = tmp_path / "worker.py" + source.write_bytes(b"import sys\nprint(sys.version_info.major)\n") + data = tmp_path / "input.txt" + data.write_bytes(b"fixed input\n") + sources = {"worker-code": source, "input-data": data} + pins = [] + for role, path in sorted(sources.items()): + payload = path.read_bytes() + pins.append({ + "role": role, + "artifact": {"location": "source/" + role, "sha256": sha256(payload).hexdigest(), + "size_bytes": len(payload), "media_type": "text/plain"}, + "format_schema": {"name": "utf8-text", "revision": "1"}, + "required_for": ["prepare", "execute"], + }) + protocol["resources"] = pins + protocol["worker_recipe"]["code_sha256"] = sha256(source.read_bytes()).hexdigest() + opened = [] + + def resources(role): + stream = sources[role].open("rb") + opened.append(stream) + return stream + + service = benchmark.PreparationService( + protocol, resources=resources, worker_role="worker-code", + ) + request = { + "schema_version": 1, "task": protocol["task"], "operation": "evaluate", + "validation_scope": "p1", "input_pins": [], + "task_parameters": deepcopy(protocol["task_parameters"]), + "comparison_conditions": {}, "expected_candidate": None, "budget_policy_refs": [], + } + archive = tmp_path / "archive" + archive.mkdir() + config = { + "schema_version": 1, "archive_namespace": "apps", "harness_namespace": "test", + "mounts": [{"prefix": "archives/local", "namespace": "apps", + "root": str(archive), "kind": "directory"}, + {"prefix": "stores/test", "namespace": "test", + "root": str(tmp_path / "store"), "kind": "harness_sqlite"}], + } + return service, request, sources, protocol, opened, config, archive + + +def test_real_prepare_freezes_exact_supplied_bytes_and_isolates_caller_mutations(preparation): + service, request, sources, protocol, opened, _, _ = preparation + protocol["task_parameters"]["label"] = "changed after configuration" + prepared, data, reader = service.prepare(request) + assert all(stream.closed for stream in opened) + benchmark.validate_prepared(prepared, data) + verify_protocol(prepared, data) + assert decode(data)["task_parameters"] == request["task_parameters"] + assert prepared["recipe"]["code_sha256"] == sha256( + sources["worker-code"].read_bytes() + ).hexdigest() + with reader("input-data") as stream: + assert stream.read() == b"fixed input\n" + with pytest.raises(KeyError): + reader("not-in-closure") + original_id = prepared["prepared_id"] + prepared["recipe"]["args"].append("caller mutation") + assert service.prepare(request)[0]["prepared_id"] == original_id + + +def test_two_layer_locations_and_ref_shaped_parameters(preparation): + service, request, _, _, _, _, _ = preparation + prepared, data, _ = service.prepare(request) + moved = deepcopy(prepared) + moved["protocol"]["location"] = "relocated/protocol.json" + for resource in moved["resources"]: + resource["artifact"]["location"] = "relocated/" + resource["role"] + benchmark.validate_prepared(moved, data) + assert benchmark.prepared_id(moved) == prepared["prepared_id"] + assert benchmark.encode(moved) != benchmark.encode(prepared) + + internal = decode(data) + internal["resources"][0]["artifact"]["location"] = "inside/new-location" + internal_bytes = encode(internal) + assert internal_bytes != data + assert benchmark.protocol_digest(internal) == prepared["protocol_digest"] + moved["protocol"].update(sha256=sha256(internal_bytes).hexdigest(), + size_bytes=len(internal_bytes)) + moved["prepared_id"] = benchmark.prepared_id(moved) + assert moved["prepared_id"] != prepared["prepared_id"] + benchmark.validate_prepared(moved, internal_bytes) + + # Identity projection does not recursively strip a location from Parameters. + projected = decode(data) + projected["task_parameters"]["ref_shaped"] = deepcopy(prepared["protocol"]) + first = benchmark.protocol_digest(projected) + projected["task_parameters"]["ref_shaped"]["location"] = "semantic/location" + assert benchmark.protocol_digest(projected) != first + changed = decode(data) + changed["task_parameters"]["label"] = "different frozen condition" + assert benchmark.protocol_digest(changed) != prepared["protocol_digest"] + request["task_parameters"]["label"] = changed["task_parameters"]["label"] + new_prepared, new_data, _ = service.prepare(request) + assert new_data != data + for key in ("prepared_id", "protocol_digest"): + assert new_prepared[key] != prepared[key] + assert new_prepared["protocol"]["sha256"] != prepared["protocol"]["sha256"] + + +@pytest.mark.parametrize("change,code", [ + (lambda value: value.update(schema_version=2), "UnsupportedSchema"), + (lambda value: value["task"].update(revision="unsupported"), "UnsupportedRuleVersion"), + (lambda value: value.update(operation="other"), "UnsupportedOperation"), + (lambda value: value.update(unrecognized=True), "InvalidRequest"), + (lambda value: value["task_parameters"].update(decimal_fixture="1.0"), "InvalidRequest"), +]) +def test_request_rejections_are_wire_errors_before_any_archive(preparation, change, code): + service, request, _, _, opened, _, archive = preparation + change(request) + response = service.prepare(request) + assert response["code"] == code and response["stage"] == "prepare" + assert response["latest_record_ref"] is None and response["retained_refs"] == [] + assert list(archive.iterdir()) == [] and opened == [] + + +@pytest.mark.parametrize("stage", ["prepare", "archive"]) +def test_t36_resource_loss_at_each_boundary(preparation, tmp_path, stage): + service, request, sources, _, _, config, archive = preparation + if stage == "prepare": + sources["input-data"].unlink() + producer = service.prepare + else: + response = service.prepare(request) + sources["input-data"].unlink() + def producer(value): + return response + result = apps_prepare( + {"schema_version": 1, "request": request}, workspace=tmp_path, + storage_config=config, prepare_benchmark=producer, + encode_prepared=benchmark.encode, validate_prepared=benchmark.validate_prepared, + ) + assert result.result["reason_code"] == "ArtifactUnavailable", result.result + assert result.result["error"]["stage"] == ("prepare" if stage == "prepare" + else "prepared_archive") + assert result.result["prepared_ref"] is None + assert list(archive.iterdir()) == [] + + +def test_wrong_resource_bytes_are_not_frozen_as_a_different_pin(preparation): + service, request, sources, _, _, _, _ = preparation + sources["worker-code"].write_bytes(b"unexpected replacement worker") + assert service.prepare(request)["code"] == "ArtifactIntegrityError" + + +@pytest.mark.parametrize("change", [ + lambda value: value["candidate_contract"].update(unknown=True), + lambda value: value["recipe"].update(unrecognized=True), + lambda value: value["recipe"]["outputs"][0].update(required=1), + lambda value: value.update(created_at="2026-02-30T00:00:00Z"), + lambda value: value.update(created_at=True), + lambda value: value.update(created_at="2026-11-20T00:00:00.1234567890Z"), + lambda value: value["resources"].append(deepcopy(value["resources"][0])), + lambda value: value["protocol"].update(location="../escape"), + lambda value: value["required_evidence"][0].update(matching_rule_revision="other"), +]) +def test_closed_prepared_shape_rejected_even_with_recomputed_identity(preparation, change): + service, request, _, _, _, _, _ = preparation + prepared, _, _ = service.prepare(request) + change(prepared) + with pytest.raises(ValueError): + prepared["prepared_id"] = benchmark.prepared_id(prepared) + benchmark.encode(prepared) + + +def test_source_change_after_preparation_is_rejected_by_independent_apps_verification( + preparation, tmp_path, +): + service, request, sources, _, _, config, archive = preparation + response = service.prepare(request) + sources["input-data"].write_bytes(b"changed input") + result = apps_prepare( + {"schema_version": 1, "request": request}, workspace=tmp_path, + storage_config=config, prepare_benchmark=lambda value: response, + encode_prepared=benchmark.encode, validate_prepared=benchmark.validate_prepared, + ) + assert result.result["reason_code"] == "ArtifactIntegrityError" + assert result.result["prepared_ref"] is None and list(archive.iterdir()) == [] + + +def test_frozen_task_rules_are_not_request_parameters(preparation): + _, _, sources, protocol, _, _, _ = preparation + protocol["claim_definitions"][0]["decision_rule"]["revision"] = "unimplemented" + with pytest.raises(benchmark.PreparationError) as error: + benchmark.PreparationService( + protocol, worker_role="worker-code", resources=lambda role: sources[role].open("rb"), + ) + assert error.value.code == "UnsupportedRuleVersion" + + +@pytest.mark.parametrize("failure", [ValueError("host bug"), CanonicalError("host", "host bug")]) +def test_programming_error_in_a_host_capability_is_not_a_port_rejection(preparation, failure): + _, request, _, protocol, _, _, _ = preparation + + def broken(role): + raise failure + + service = benchmark.PreparationService(protocol, resources=broken, worker_role="worker-code") + with pytest.raises(ValueError) as error: + service.prepare(request) + assert error.value is failure + + +def test_t36_real_port_archive_and_cross_process_fixed_readback(preparation, tmp_path): + service, request, sources, _, _, config, _ = preparation + expected, protocol, _ = service.prepare(request) + result = apps_prepare( + {"schema_version": 1, "request": request}, workspace=tmp_path, + storage_config=config, prepare_benchmark=service.prepare, + encode_prepared=benchmark.encode, validate_prepared=benchmark.validate_prepared, + ) + assert result.result["exit_code"] == 0, result.result + ref = result.result["prepared_ref"] + assert ref["prepared_id"] == expected["prepared_id"] + archived = read_prepared(config, ref) + benchmark.validate_prepared(archived, protocol) + child = """ +import json, sys +sys.path.insert(0, sys.argv[1]) +from inferrsi.apps.storage.archive import read_prepared, read_archive_artifact +from inferrsi.benchmark.preparation import encode, validate_prepared +config, ref = json.loads(sys.stdin.read()) +prepared = read_prepared(config, ref) +protocol = read_archive_artifact(config, prepared['protocol']) +validate_prepared(prepared, protocol) +print(json.dumps({'prepared': encode(prepared).hex(), 'protocol': protocol.hex(), + 'resources': {x['role']: read_archive_artifact(config, x['artifact']).hex() + for x in prepared['resources']}})) +""" + process = subprocess.run( + [sys.executable, "-I", "-c", child, str(ROOT / "src")], cwd=tmp_path, + env={**os.environ, "PYTHONUTF8": "1"}, input=json.dumps([config, ref]), + capture_output=True, text=True, timeout=30, + ) + assert process.returncode == 0, process.stderr + observed = json.loads(process.stdout) + prepared_bytes = bytes.fromhex(observed["prepared"]) + assert sha256(prepared_bytes).hexdigest() == ref["artifact"]["sha256"] + assert len(prepared_bytes) == ref["artifact"]["size_bytes"] + assert observed["protocol"] == protocol.hex() + assert observed["resources"] == {role: path.read_bytes().hex() + for role, path in sources.items()} From 3a19e5ae9dd718d3d2f4538a150bebd4242256c2 Mon Sep 17 00:00:00 2001 From: zengchuang Date: Fri, 20 Nov 2026 09:46:32 +0800 Subject: [PATCH 2/5] fix(benchmark): reject invalid candidate metadata before reading resources Agent-assisted-by: Codex (author) Signed-off-by: zengchuang --- src/inferrsi/benchmark/preparation.py | 23 ++++++++++++++--------- tests/test_benchmark_preparation.py | 6 ++++++ 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/src/inferrsi/benchmark/preparation.py b/src/inferrsi/benchmark/preparation.py index 7b187e975..e99c06e1b 100644 --- a/src/inferrsi/benchmark/preparation.py +++ b/src/inferrsi/benchmark/preparation.py @@ -257,6 +257,18 @@ def _protocol(protocol: dict) -> None: _definition(ref) +def _expectation(candidate, contract: dict) -> None: + if candidate is None: + return + _object(candidate, {"candidate_id", "sha256", "format_schema"}) + _require(_id(candidate["candidate_id"]) and isinstance(candidate["sha256"], str) + and _SHA.fullmatch(candidate["sha256"]) is not None, + "Invalid expected candidate.") + _definition(candidate["format_schema"]) + _require(candidate["format_schema"] == contract["format_schema"], + "Expected candidate format differs from the frozen contract.") + + def _prepared(prepared: dict) -> None: canonical_encode(prepared) _object(prepared, _PREPARED) @@ -289,15 +301,7 @@ def _prepared(prepared: dict) -> None: _resource(resource) roles = [resource["role"] for resource in prepared["resources"]] _require(roles == sorted(set(roles)), "Prepared closure must be sorted by unique role.") - candidate = prepared["expected_candidate"] - if candidate is not None: - _object(candidate, {"candidate_id", "sha256", "format_schema"}) - _require(_id(candidate["candidate_id"]) and isinstance(candidate["sha256"], str) - and _SHA.fullmatch(candidate["sha256"]) is not None, - "Invalid expected candidate.") - _definition(candidate["format_schema"]) - _require(candidate["format_schema"] == prepared["candidate_contract"]["format_schema"], - "Expected candidate format differs from the frozen contract.") + _expectation(prepared["expected_candidate"], prepared["candidate_contract"]) _require(prepared["prepared_id"] == prepared_id(prepared), "Prepared identity does not match its content.", "IdentityMismatch") @@ -394,6 +398,7 @@ def _prepare(self, request: dict) -> tuple: _parameters(request["task_parameters"]) _require(request["comparison_conditions"] == {}, "The first P1 task uses the configured frozen comparison conditions.") + _expectation(request["expected_candidate"], protocol["candidate_contract"]) protocol["task_parameters"] = deepcopy(request["task_parameters"]) protocol["budget_policy_refs"] = deepcopy(request["budget_policy_refs"]) direct = {item["role"]: item for item in protocol["resources"]} diff --git a/tests/test_benchmark_preparation.py b/tests/test_benchmark_preparation.py index d85cfa365..9913c3c6f 100644 --- a/tests/test_benchmark_preparation.py +++ b/tests/test_benchmark_preparation.py @@ -147,6 +147,12 @@ def test_two_layer_locations_and_ref_shaped_parameters(preparation): (lambda value: value.update(operation="other"), "UnsupportedOperation"), (lambda value: value.update(unrecognized=True), "InvalidRequest"), (lambda value: value["task_parameters"].update(decimal_fixture="1.0"), "InvalidRequest"), + (lambda value: value.update(expected_candidate={"candidate_id": "incomplete"}), + "InvalidRequest"), + (lambda value: value.update(expected_candidate={ + "candidate_id": "candidate", "sha256": "a" * 64, + "format_schema": {"name": "other", "revision": "1"}, + }), "InvalidRequest"), ]) def test_request_rejections_are_wire_errors_before_any_archive(preparation, change, code): service, request, _, _, opened, _, archive = preparation From 73d1e446e3a877d801c7ff1377a91c3e508b75a4 Mon Sep 17 00:00:00 2001 From: zengchuang Date: Fri, 20 Nov 2026 09:53:44 +0800 Subject: [PATCH 3/5] fix(benchmark): preserve host resource capability failures Agent-assisted-by: Codex (author) Signed-off-by: zengchuang --- docs/contracts/production-prepare.md | 3 +++ src/inferrsi/benchmark/preparation.py | 3 ++- tests/test_benchmark_preparation.py | 38 ++++++++++++++++++++++++++- 3 files changed, 42 insertions(+), 2 deletions(-) diff --git a/docs/contracts/production-prepare.md b/docs/contracts/production-prepare.md index 0b88a1495..a55ffb8e2 100644 --- a/docs/contracts/production-prepare.md +++ b/docs/contracts/production-prepare.md @@ -28,6 +28,9 @@ stream. It does not interpret a request-provided location as a filesystem path. The frozen resource metadata is verified during preparation. The returned `open_resource` accepts only roles in that preparation's closure and reopens the resource through the same capability. It does not write or archive bytes. +Provisional source locations may alias: byte authority is the configured role, +not a lookup or de-duplication by that location. Apps assigns a distinct final +archive path to each role and verifies the bytes against that role's metadata. The first task keeps the P1 candidate, claims, evidence rules, worker recipe shape and sampling conditions. PrepareRequest can bind its candidate expectation, diff --git a/src/inferrsi/benchmark/preparation.py b/src/inferrsi/benchmark/preparation.py index e99c06e1b..def47cbe6 100644 --- a/src/inferrsi/benchmark/preparation.py +++ b/src/inferrsi/benchmark/preparation.py @@ -347,7 +347,8 @@ def _measure(source) -> tuple[int, str]: with closing(source): while True: chunk = source.read(1024 * 1024) - _require(isinstance(chunk, bytes), "Resource stream must return bytes.") + if not isinstance(chunk, bytes): + raise TypeError("A host resource capability must return binary bytes.") if not chunk: break size += len(chunk) diff --git a/tests/test_benchmark_preparation.py b/tests/test_benchmark_preparation.py index 9913c3c6f..c229bd7fe 100644 --- a/tests/test_benchmark_preparation.py +++ b/tests/test_benchmark_preparation.py @@ -6,13 +6,14 @@ import sys from copy import deepcopy from hashlib import sha256 +from io import StringIO from pathlib import Path import pytest from inferrsi.apps.api import prepare as apps_prepare from inferrsi.apps.handoff.bridge import verify_protocol -from inferrsi.apps.storage.archive import read_prepared +from inferrsi.apps.storage.archive import read_archive_artifact, read_prepared from inferrsi.benchmark import preparation as benchmark from inferrsi.contracts.json_v1 import CanonicalError, decode, encode @@ -250,6 +251,41 @@ def broken(role): assert error.value is failure +def test_text_stream_is_a_closed_host_capability_error(preparation): + _, request, _, protocol, _, _, _ = preparation + stream = StringIO("a text capability is not a binary resource") + service = benchmark.PreparationService( + protocol, resources=lambda role: stream, worker_role="worker-code", + ) + with pytest.raises(TypeError, match="binary bytes"): + service.prepare(request) + assert stream.closed + + +def test_provisional_locations_do_not_replace_role_authority_or_archive_paths( + preparation, tmp_path, +): + service, request, sources, protocol, _, config, _ = preparation + pin = deepcopy(protocol["resources"][0]) + assert pin["role"] == "input-data" + pin["artifact"]["location"] = protocol["resources"][1]["artifact"]["location"] + request["input_pins"] = [pin] + result = apps_prepare( + {"schema_version": 1, "request": request}, workspace=tmp_path, + storage_config=config, prepare_benchmark=service.prepare, + encode_prepared=benchmark.encode, validate_prepared=benchmark.validate_prepared, + ) + assert result.result["exit_code"] == 0, result.result + prepared = read_prepared(config, result.result["prepared_ref"]) + assert len({item["artifact"]["location"] for item in prepared["resources"]}) == 2 + assert {item["role"]: read_archive_artifact(config, item["artifact"]) + for item in prepared["resources"]} == { + role: path.read_bytes() for role, path in sources.items() + } + frozen = decode(read_archive_artifact(config, prepared["protocol"])) + assert len({item["artifact"]["location"] for item in frozen["resources"]}) == 1 + + def test_t36_real_port_archive_and_cross_process_fixed_readback(preparation, tmp_path): service, request, sources, _, _, config, _ = preparation expected, protocol, _ = service.prepare(request) From a1d441ae8b758aa603b684b23b3222e23c64c750 Mon Sep 17 00:00:00 2001 From: zengchuang Date: Fri, 20 Nov 2026 09:59:37 +0800 Subject: [PATCH 4/5] test(benchmark): make preparation codec rejection checks explicit Agent-assisted-by: Codex (author) Signed-off-by: zengchuang --- docs/contracts/production-prepare.md | 8 ++++++-- src/inferrsi/benchmark/preparation.py | 3 ++- tests/test_benchmark_preparation.py | 11 +++++++++-- 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/docs/contracts/production-prepare.md b/docs/contracts/production-prepare.md index a55ffb8e2..f87ff1801 100644 --- a/docs/contracts/production-prepare.md +++ b/docs/contracts/production-prepare.md @@ -23,8 +23,10 @@ prepared, protocol_bytes, open_resource = service.prepare(request) ``` `open_fixed_resource(role)` is a host-supplied capability returning bytes or a -fresh binary stream for an authorized role. The caller owns closing each returned -stream. It does not interpret a request-provided location as a filesystem path. +fresh binary stream for an authorized role. Preparation closes each stream it +opens for verification; after prepare, the closure's consumer (normally Apps) +owns closing each stream it opens. The opener does not interpret a +request-provided location as a filesystem path. The frozen resource metadata is verified during preparation. The returned `open_resource` accepts only roles in that preparation's closure and reopens the resource through the same capability. It does not write or archive bytes. @@ -36,6 +38,8 @@ The first task keeps the P1 candidate, claims, evidence rules, worker recipe shape and sampling conditions. PrepareRequest can bind its candidate expectation, P1 task parameters, budget references and additional resource pins. A supplied pin cannot replace a frozen role's hash, format or required use. The first task +does allow that pin to replace the role's provisional source location; this +changes the frozen protocol bytes, with identity effects described below. The first task accepts an empty `comparison_conditions`: comparison conditions are configured in the frozen task definition, rather than silently accepting unknown overrides. Unknown task, grader, worker or claim-rule revisions are refused. diff --git a/src/inferrsi/benchmark/preparation.py b/src/inferrsi/benchmark/preparation.py index def47cbe6..ddcd5ec0a 100644 --- a/src/inferrsi/benchmark/preparation.py +++ b/src/inferrsi/benchmark/preparation.py @@ -360,7 +360,8 @@ class PreparationService: """The supported P1 prepare port bound to frozen rules and read capabilities. ``worker_role`` identifies the retained code bytes for recipe.code_sha256. - Each opener call gives its stream to the caller, which owns closing it. + Preparation closes every stream it opens for verification. After prepare, + the consumer of the returned resource reader owns each stream it opens. The returned closure reopens resources: deletion fails at the Apps archive boundary and changed bytes are independently rejected by Apps' verifier. """ diff --git a/tests/test_benchmark_preparation.py b/tests/test_benchmark_preparation.py index c229bd7fe..39a8ebabb 100644 --- a/tests/test_benchmark_preparation.py +++ b/tests/test_benchmark_preparation.py @@ -201,15 +201,22 @@ def test_wrong_resource_bytes_are_not_frozen_as_a_different_pin(preparation): lambda value: value.update(created_at=True), lambda value: value.update(created_at="2026-11-20T00:00:00.1234567890Z"), lambda value: value["resources"].append(deepcopy(value["resources"][0])), - lambda value: value["protocol"].update(location="../escape"), lambda value: value["required_evidence"][0].update(matching_rule_revision="other"), ]) def test_closed_prepared_shape_rejected_even_with_recomputed_identity(preparation, change): service, request, _, _, _, _, _ = preparation prepared, _, _ = service.prepare(request) change(prepared) + prepared["prepared_id"] = benchmark.prepared_id(prepared) with pytest.raises(ValueError): - prepared["prepared_id"] = benchmark.prepared_id(prepared) + benchmark.encode(prepared) + + +def test_unsafe_location_is_rejected_by_the_encoder_itself(preparation): + service, request, _, _, _, _, _ = preparation + prepared, _, _ = service.prepare(request) + prepared["protocol"]["location"] = "../escape" + with pytest.raises(benchmark.PreparationError, match="logical artifact location"): benchmark.encode(prepared) From 0a82e7b5ad7115f1b2e81818bcf227129208e4ec Mon Sep 17 00:00:00 2001 From: zengchuang Date: Fri, 20 Nov 2026 10:13:03 +0800 Subject: [PATCH 5/5] fix(benchmark): register the preparation integration consumer Agent-assisted-by: Codex (author) Signed-off-by: zengchuang --- docs/apps-module-migrations.json | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/apps-module-migrations.json b/docs/apps-module-migrations.json index 9090a5956..503abf3be 100644 --- a/docs/apps-module-migrations.json +++ b/docs/apps-module-migrations.json @@ -36,6 +36,7 @@ "tests/test_apps_prepare.py", "tests/test_apps_publication.py", "tests/test_apps_replay.py", + "tests/test_benchmark_preparation.py", "tests/test_result_delivery.py" ] }, @@ -137,7 +138,8 @@ "tests/test_apps_evidence.py", "tests/test_apps_execution_binding.py", "tests/test_apps_generation_plan.py", - "tests/test_apps_module_migrations.py" + "tests/test_apps_module_migrations.py", + "tests/test_benchmark_preparation.py" ] }, {