diff --git a/Makefile b/Makefile index 60b09a699..ce5b0a57f 100644 --- a/Makefile +++ b/Makefile @@ -60,6 +60,7 @@ e2e-test: ## Run CLI to Client SDK to Server end-to-end tests. evaluation-unit-test: ## Run the evaluation project's unit tests, the work-continuity benchmark included. @uv sync --project evaluation --frozen @uv run --project evaluation pytest -c evaluation/pyproject.toml \ + evaluation/artifact_evolution/tests \ evaluation/coding/swebench_pro/tests/unit \ evaluation/coding/work_continuity/tests/unit \ evaluation/memory/longmemeval_v2/tests -m "not live" -q diff --git a/evaluation/README.md b/evaluation/README.md index 7a7491c2d..e855f1308 100644 --- a/evaluation/README.md +++ b/evaluation/README.md @@ -10,11 +10,16 @@ Python environment and build configuration. | [`memory/`](memory/README.md) | Memory retrieval, answer quality, and write-gate evidence | [LoCoMo](memory/locomo/README.md), [LoCoMo-Plus](memory/locomo_plus/README.md), [LongMemEval-V2](memory/longmemeval_v2/README.md), [Memory write gate](memory/write_gate/README.md) | | [`performance/`](performance/README.md) | Load, latency, storage growth, and capacity limits | None yet | | [`skills/`](skills/README.md) | Agent instruction routing and authorization regressions | [Claude Code skill-up](skills/skill-up/README.md), [ZCode native CLI](zcode_guidance/README.md) | +| [`artifact_evolution/`](artifact_evolution/README.md) | Evaluation-owned Dream recording and faithful replay | [Artifact evolution](artifact_evolution/README.md) | ```text evaluation/ ├── pyproject.toml # Shared powercontext-eval environment and build ├── uv.lock +├── artifact_evolution/ # Artifact-evolution recording and faithful replay +│ ├── src/powercontext_eval_artifact_evolution/ +│ ├── tests/ +│ └── README.md ├── coding/ │ ├── swebench_pro/ │ │ ├── src/powercontext_eval_swebench_pro/ @@ -89,9 +94,10 @@ uv run --directory evaluation ty check uv build --project evaluation ``` -The `powercontext-eval` distribution contains three packages: `powercontext_eval_swebench_pro`, -`powercontext_eval_work_continuity`, and `powercontext_eval_longmemeval_v2`. Editable installs and wheels expose the -independent `swebench-pro`, `work-continuity`, and `longmemeval-v2` commands and their suite modules. +The `powercontext-eval` distribution contains four packages: `powercontext_eval_artifact_evolution`, +`powercontext_eval_swebench_pro`, `powercontext_eval_work_continuity`, and `powercontext_eval_longmemeval_v2`. +Editable installs and wheels expose the independent `artifact-evolution`, `swebench-pro`, `work-continuity`, and +`longmemeval-v2` commands and their suite modules. LoCoMo behavior tests remain under [`tests/`](../tests/), including [`tests/evaluation/`](../tests/evaluation/). Skill-up and ZCode guidance keep their own validation tests. The [Bub replay harness](../e2e/bub/README.md) and [product E2E tests](../tests/e2e/) cover cross-component acceptance. diff --git a/evaluation/artifact_evolution/README.md b/evaluation/artifact_evolution/README.md new file mode 100644 index 000000000..be0622c8a --- /dev/null +++ b/evaluation/artifact_evolution/README.md @@ -0,0 +1,60 @@ +# Artifact-evolution evaluation + +This suite owns the evaluation-only recording and faithful replay contract for +Experience Dream evolution. It does not extend Runtime, add database tables, or +create Candidates. The bundle stores exact references, digests, realized +results, deterministic validation results, nomination state, and saved cost. + +The versioned bundle is `powercontext.artifact-evolution/v1`. Controlled capture +is explicit; ordinary production capture is not enabled by this suite. Prompt, +task, and evidence bodies are not part of the contract. Optional output payloads +require either the deterministic `redacted` policy (sensitive keys and +credential-shaped strings are replaced before persistence) or an `approved` +policy with an explicit `approval_ref`. The recorder stores a structured +`scope_id`/`run_id` reference to the terminal Dream run; it does not copy the +Runtime row or create a Candidate. + +## Commands + +Validate a bundle without calling a model or writing Runtime state: + +```bash +uv run --project evaluation artifact-evolution validate path/to/bundle.json +``` + +Replay one exact action using the saved result only: + +```bash +uv run --project evaluation artifact-evolution replay path/to/bundle.json \ + --decision-id decision-1 \ + --trajectory-id trajectory-1 \ + --observations-digest 0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef \ + --action-signature refine-experience:v1 \ + --ordering 0 +``` + +Report exact replay coverage and saved generation cost: + +```bash +uv run --project evaluation artifact-evolution coverage path/to/bundle.json +``` + +Record one controlled terminal `refine_experience` Dream snapshot into a new +bundle. The input is a JSON serialization of the existing Runtime `DreamRun` +shape; the evaluation package only reads the fields it needs and computes the +observation and output digests. + +```bash +uv run --project evaluation artifact-evolution record \ + base-bundle.json dream-run.json recorded-bundle.json \ + --action-signature refine-experience:v1 \ + --policy-id baseline-policy-v1 \ + --generator-id dream.refine_experience \ + --runtime-version runtime-v1 +``` + +An absent or mismatched action returns `out_of_support`. Replay never runs a +generator or evaluator and never mutates Runtime, Candidate, Review, or Artifact +state. Exact replay support is identity-level: the trajectory, decision id, +observation digest, action signature, and ordering must all match. Coverage +therefore reports unsupported decision ids and is not a cross-policy comparison. diff --git a/evaluation/artifact_evolution/ruff.toml b/evaluation/artifact_evolution/ruff.toml new file mode 100644 index 000000000..01dae24b3 --- /dev/null +++ b/evaluation/artifact_evolution/ruff.toml @@ -0,0 +1,16 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +target-version = "py311" +line-length = 120 diff --git a/evaluation/artifact_evolution/src/powercontext_eval_artifact_evolution/__init__.py b/evaluation/artifact_evolution/src/powercontext_eval_artifact_evolution/__init__.py new file mode 100644 index 000000000..9be01ebb7 --- /dev/null +++ b/evaluation/artifact_evolution/src/powercontext_eval_artifact_evolution/__init__.py @@ -0,0 +1,59 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Evaluation-owned recording and faithful replay for Artifact evolution.""" + +from powercontext_eval_artifact_evolution.bundle import ( + BundleIOError, + BundleValidationError, + load_bundle, + record_attempt, + write_bundle, +) +from powercontext_eval_artifact_evolution.models import ( + ArtifactEvolutionBundle, + DecisionRequest, + DreamRunRef, + ProposalAttempt, +) +from powercontext_eval_artifact_evolution.recording import ( + DreamRecordingError, + DreamRunSnapshot, + decision_for_dream_run, + record_dream_attempt, + snapshot_from_dream_run, +) +from powercontext_eval_artifact_evolution.replay import ( + ReplayEngine, + ReplayResult, +) + +__all__ = [ + "ArtifactEvolutionBundle", + "BundleIOError", + "BundleValidationError", + "DecisionRequest", + "DreamRecordingError", + "DreamRunRef", + "DreamRunSnapshot", + "ProposalAttempt", + "ReplayEngine", + "ReplayResult", + "decision_for_dream_run", + "load_bundle", + "record_attempt", + "record_dream_attempt", + "snapshot_from_dream_run", + "write_bundle", +] diff --git a/evaluation/artifact_evolution/src/powercontext_eval_artifact_evolution/bundle.py b/evaluation/artifact_evolution/src/powercontext_eval_artifact_evolution/bundle.py new file mode 100644 index 000000000..db38c23c6 --- /dev/null +++ b/evaluation/artifact_evolution/src/powercontext_eval_artifact_evolution/bundle.py @@ -0,0 +1,116 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Read, write, and append operations for artifact-evolution bundles.""" + +from __future__ import annotations + +import json +import os +import tempfile +from pathlib import Path + +from pydantic import ValidationError + +from powercontext_eval_artifact_evolution.models import ArtifactEvolutionBundle, ProposalAttempt +from powercontext_eval_artifact_evolution.privacy import digest_json, redact_payload + + +class BundleIOError(ValueError): + """A bundle could not be read or written without losing its contract.""" + + +class BundleValidationError(BundleIOError): + """A bundle is not valid `powercontext.artifact-evolution/v1` JSON.""" + + +def load_bundle(path: Path) -> ArtifactEvolutionBundle: + """Load and validate a UTF-8 JSON bundle without invoking any evaluator.""" + + resolved = path.resolve() + try: + raw = resolved.read_bytes() + except OSError as error: + raise BundleIOError(f"cannot read artifact-evolution bundle: {resolved}") from error + if not raw.strip(): + raise BundleValidationError(f"artifact-evolution bundle is blank: {resolved}") + try: + value = json.loads(raw.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as error: + raise BundleValidationError("artifact-evolution bundle must be UTF-8 JSON") from error + if not isinstance(value, dict): + raise BundleValidationError("artifact-evolution bundle must be a JSON object") + try: + return ArtifactEvolutionBundle.model_validate_json(raw) + except ValidationError as error: + raise BundleValidationError(f"invalid artifact-evolution bundle: {error}") from error + + +def write_bundle(bundle: ArtifactEvolutionBundle, path: Path) -> Path: + """Write one bundle, refusing to overwrite a previously recorded artifact.""" + + target = path.resolve() + try: + bundle = ArtifactEvolutionBundle.model_validate(bundle.model_dump(mode="python", by_alias=True)) + except ValidationError as error: + raise BundleValidationError(f"cannot write invalid artifact-evolution bundle: {error}") from error + temporary: Path | None = None + try: + target.parent.mkdir(parents=True, exist_ok=True) + descriptor, temporary_name = tempfile.mkstemp(prefix=f".{target.name}.", dir=target.parent) + temporary = Path(temporary_name) + with os.fdopen(descriptor, "w", encoding="utf-8") as stream: + stream.write(_render(bundle)) + stream.flush() + os.fsync(stream.fileno()) + # Hard-linking the completed file gives create-without-overwrite semantics. + os.link(temporary, target) + except FileExistsError as error: + raise BundleIOError(f"refusing to overwrite artifact-evolution bundle: {target}") from error + except OSError as error: + raise BundleIOError(f"cannot write artifact-evolution bundle: {target}") from error + finally: + if temporary is not None: + try: + temporary.unlink() + except FileNotFoundError: + pass + return target + + +def record_attempt(bundle: ArtifactEvolutionBundle, attempt: ProposalAttempt) -> ArtifactEvolutionBundle: + """Return a validated bundle with one additional realized attempt. + + This pure operation makes recording explicit and keeps the bundle immutable; + persistence remains the caller's choice through :func:`write_bundle`. + """ + + if attempt.result.output_payload is not None: + payload = attempt.result.output_payload + if bundle.capture_policy.payloads == "redacted": + payload = redact_payload(payload) + attempt = attempt.model_copy( + update={ + "result": attempt.result.model_copy( + update={"output_payload": payload, "output_digest": digest_json(payload)} + ) + } + ) + updated = bundle.model_copy(update={"attempts": (*bundle.attempts, attempt)}) + return ArtifactEvolutionBundle.model_validate(updated.model_dump(mode="python", by_alias=True)) + + +def _render(bundle: ArtifactEvolutionBundle) -> str: + payload = bundle.model_dump(mode="json", by_alias=True, exclude_none=True) + return json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True) + "\n" diff --git a/evaluation/artifact_evolution/src/powercontext_eval_artifact_evolution/cli.py b/evaluation/artifact_evolution/src/powercontext_eval_artifact_evolution/cli.py new file mode 100644 index 000000000..e020dd2e7 --- /dev/null +++ b/evaluation/artifact_evolution/src/powercontext_eval_artifact_evolution/cli.py @@ -0,0 +1,181 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""CLI for validating and replaying Artifact-evolution bundles.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Annotated + +import typer +from pydantic import ValidationError + +from powercontext_eval_artifact_evolution.bundle import BundleIOError, load_bundle, write_bundle +from powercontext_eval_artifact_evolution.models import DecisionRequest +from powercontext_eval_artifact_evolution.recording import ( + DreamRecordingError, + DreamRunSnapshot, + decision_for_dream_run, + record_dream_attempt, +) +from powercontext_eval_artifact_evolution.replay import ReplayEngine + +app = typer.Typer(no_args_is_help=True, help="Validate and faithfully replay Artifact-evolution bundles.") + + +@app.command("validate") +def validate_bundle(path: Annotated[Path, typer.Argument()]) -> None: + """Validate one bundle without calling a model or writing any Runtime state.""" + + try: + bundle = load_bundle(path) + except BundleIOError as error: + typer.echo(f"Artifact-evolution validation failed: {error}", err=True) + raise typer.Exit(code=1) from None + typer.echo( + json.dumps( + { + "classification": "artifact-evolution-bundle-validation", + "schema": bundle.schema_version, + "bundle_id": bundle.bundle_id, + "operation": bundle.workload.operation, + "requested_decisions": len(bundle.requested_decisions), + "recorded_attempts": len(bundle.attempts), + }, + ensure_ascii=False, + sort_keys=True, + ) + ) + + +@app.command("replay") +def replay_bundle( + path: Annotated[Path, typer.Argument()], + decision_id: Annotated[str, typer.Option("--decision-id")], + trajectory_id: Annotated[str, typer.Option("--trajectory-id")], + observations_digest: Annotated[str, typer.Option("--observations-digest")], + action_signature: Annotated[str, typer.Option("--action-signature")], + ordering: Annotated[int, typer.Option("--ordering", min=0)], +) -> None: + """Return one exact realized result or ``out_of_support``.""" + + try: + bundle = load_bundle(path) + except BundleIOError as error: + typer.echo(f"Artifact-evolution replay failed: {error}", err=True) + raise typer.Exit(code=1) from None + try: + request = DecisionRequest( + decision_id=decision_id, + trajectory_id=trajectory_id, + observations_digest=observations_digest, + action_signature=action_signature, + ordering=ordering, + ) + result = ReplayEngine(bundle).replay(request) + except ValidationError as error: + typer.echo(f"Artifact-evolution replay failed: invalid decision request ({error.errors()[0]['msg']})", err=True) + raise typer.Exit(code=1) from None + payload: dict[str, object] = {"status": result.status} + if result.reason is not None: + payload["reason"] = result.reason + if result.attempt is not None: + payload["attempt"] = result.attempt.model_dump(mode="json", exclude_none=True) + typer.echo(json.dumps(payload, ensure_ascii=False, sort_keys=True)) + + +@app.command("coverage") +def bundle_coverage(path: Annotated[Path, typer.Argument()]) -> None: + """Report basic exact-replay coverage and saved generation cost.""" + + try: + bundle = load_bundle(path) + except BundleIOError as error: + typer.echo(f"Artifact-evolution coverage failed: {error}", err=True) + raise typer.Exit(code=1) from None + coverage = ReplayEngine(bundle).coverage() + typer.echo( + json.dumps( + { + "classification": "artifact-evolution-exact-replay-coverage", + "requested_decisions": coverage.requested_decisions, + "supported_decisions": coverage.supported_decisions, + "decision_ratio": coverage.decision_ratio, + "requested_trajectories": coverage.requested_trajectories, + "complete_trajectories": coverage.complete_trajectories, + "trajectory_ratio": coverage.trajectory_ratio, + "saved_model_calls": coverage.saved_model_calls, + "saved_retries": coverage.saved_retries, + "saved_input_tokens": coverage.saved_input_tokens, + "saved_output_tokens": coverage.saved_output_tokens, + "input_tokens_complete": coverage.input_tokens_complete, + "output_tokens_complete": coverage.output_tokens_complete, + "unsupported_decision_ids": coverage.unsupported_decision_ids, + }, + ensure_ascii=False, + sort_keys=True, + ) + ) + + +@app.command("record") +def record_bundle( + bundle_path: Annotated[Path, typer.Argument(help="Existing recording bundle to extend.")], + dream_run_path: Annotated[Path, typer.Argument(help="JSON snapshot of one terminal DreamRun.")], + output_path: Annotated[Path, typer.Argument(help="New output bundle path.")], + action_signature: Annotated[str, typer.Option("--action-signature")], + policy_id: Annotated[str, typer.Option("--policy-id")], + generator_id: Annotated[str, typer.Option("--generator-id")], + runtime_version: Annotated[str, typer.Option("--runtime-version")], + decision_id: Annotated[str | None, typer.Option("--decision-id")] = None, + trajectory_id: Annotated[str | None, typer.Option("--trajectory-id")] = None, + ordering: Annotated[int, typer.Option("--ordering", min=0)] = 0, +) -> None: + """Record one controlled terminal refine_experience Dream snapshot.""" + + try: + bundle = load_bundle(bundle_path) + snapshot = DreamRunSnapshot.model_validate_json(dream_run_path.read_bytes()) + decision = decision_for_dream_run( + snapshot, + action_signature=action_signature, + decision_id=decision_id, + trajectory_id=trajectory_id, + ordering=ordering, + ) + recorded = record_dream_attempt( + bundle, + snapshot, + decision=decision, + policy_id=policy_id, + generator_id=generator_id, + runtime_version=runtime_version, + ) + path = write_bundle(recorded, output_path) + except (BundleIOError, DreamRecordingError, ValidationError, OSError, ValueError) as error: + typer.echo(f"Artifact-evolution record failed: {error}", err=True) + raise typer.Exit(code=1) from None + typer.echo(json.dumps({"status": "recorded", "output": str(path)}, ensure_ascii=False, sort_keys=True)) + + +def main() -> None: + """Run the Artifact-evolution CLI.""" + + app() + + +if __name__ == "__main__": + main() diff --git a/evaluation/artifact_evolution/src/powercontext_eval_artifact_evolution/models.py b/evaluation/artifact_evolution/src/powercontext_eval_artifact_evolution/models.py new file mode 100644 index 000000000..815036542 --- /dev/null +++ b/evaluation/artifact_evolution/src/powercontext_eval_artifact_evolution/models.py @@ -0,0 +1,256 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Versioned data contracts for Artifact-evolution recording. + +The models deliberately contain references and digests rather than prompt, +task, or evidence bodies. They are evaluation artifacts, not Runtime records. +""" + +from __future__ import annotations + +from datetime import datetime +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field, JsonValue, field_validator, model_validator + +from powercontext_eval_artifact_evolution.privacy import digest_json + +BUNDLE_SCHEMA = "powercontext.artifact-evolution/v1" +SHA256_LENGTH = 64 + + +class _ContractModel(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True, strict=True, populate_by_name=True) + + +class CapturePolicy(_ContractModel): + """Retention policy for a controlled evaluation bundle.""" + + mode: Literal["controlled"] + payloads: Literal["none", "redacted", "approved"] = "none" + retention_class: str = Field(min_length=1) + approval_ref: str | None = None + + @model_validator(mode="after") + def _validate_approval(self) -> CapturePolicy: + if self.payloads == "approved" and not self.approval_ref: + raise ValueError("approved payload capture requires approval_ref") + if self.payloads != "approved" and self.approval_ref is not None: + raise ValueError("approval_ref is only valid for approved payload capture") + return self + + +class Budget(_ContractModel): + """Declared evaluation limits, kept separate from live Runtime budgets.""" + + max_proposal_attempts: int = Field(ge=1) + max_model_calls: int = Field(ge=1) + max_input_tokens: int | None = Field(default=None, ge=1) + max_output_tokens: int | None = Field(default=None, ge=1) + max_wall_time_ms: int | None = Field(default=None, ge=1) + max_concurrency: int = Field(default=1, ge=1) + + +class Workload(_ContractModel): + """Scope-safe workload identity and exact Dream input binding.""" + + experiment_id: str = Field(min_length=1) + scope_digest: str = Field(min_length=SHA256_LENGTH, max_length=SHA256_LENGTH) + operation: Literal["refine_experience"] + target_artifact_ref: str | None = None + expected_head: str | None = None + fixture_digest: str = Field(min_length=SHA256_LENGTH, max_length=SHA256_LENGTH) + treatment: str = Field(min_length=1) + upstream_task_ref: str | None = None + evidence_refs: tuple[str, ...] = () + declared_budget: Budget + + @field_validator("scope_digest", "fixture_digest") + @classmethod + def _require_sha256(cls, value: str) -> str: + if any(character not in "0123456789abcdef" for character in value.lower()): + raise ValueError("digest must be lowercase hexadecimal") + return value.lower() + + +class DecisionRequest(_ContractModel): + """One requested decision and its canonical baseline action.""" + + decision_id: str = Field(min_length=1) + trajectory_id: str = Field(min_length=1) + observations_digest: str = Field(min_length=SHA256_LENGTH, max_length=SHA256_LENGTH) + action_signature: str = Field(min_length=1) + ordering: int = Field(ge=0) + + @field_validator("observations_digest") + @classmethod + def _require_sha256(cls, value: str) -> str: + if any(character not in "0123456789abcdef" for character in value.lower()): + raise ValueError("observations_digest must be lowercase hexadecimal") + return value.lower() + + @property + def key(self) -> tuple[str, str, str, str, int]: + return (self.trajectory_id, self.decision_id, self.observations_digest, self.action_signature, self.ordering) + + +class TrajectoryRequest(_ContractModel): + """The ordered decisions requested for one replay trajectory.""" + + trajectory_id: str = Field(min_length=1) + decision_ids: tuple[str, ...] = Field(min_length=1) + + +class GenerationIdentity(_ContractModel): + """Identities needed to interpret a realized generation result.""" + + policy_id: str = Field(min_length=1) + generator_id: str = Field(min_length=1) + model_id: str = Field(min_length=1) + prompt_schema_version: str = Field(min_length=1) + runtime_version: str = Field(min_length=1) + + +class ValidationResult(_ContractModel): + """One deterministic validator/scorer result saved with an attempt.""" + + validator_id: str = Field(min_length=1) + version: str = Field(min_length=1) + status: Literal["passed", "failed", "not_run"] + details: dict[str, JsonValue] = Field(default_factory=dict) + + +class AttemptResult(_ContractModel): + """Typed terminal result with a digest of any generated output.""" + + outcome: Literal["proposed", "no_change", "needs_evidence", "error"] + output_digest: str | None = Field(default=None, min_length=SHA256_LENGTH, max_length=SHA256_LENGTH) + output_payload: dict[str, JsonValue] | None = None + error_code: str | None = None + + @field_validator("output_digest") + @classmethod + def _require_sha256(cls, value: str | None) -> str | None: + if value is not None and any(character not in "0123456789abcdef" for character in value.lower()): + raise ValueError("output_digest must be lowercase hexadecimal") + return None if value is None else value.lower() + + @model_validator(mode="after") + def _validate_result(self) -> AttemptResult: + if self.outcome == "proposed" and self.output_digest is None: + raise ValueError("proposed results require output_digest") + if self.outcome == "error" and not self.error_code: + raise ValueError("error results require error_code") + return self + + +class AttemptCost(_ContractModel): + """Saved cost of one realized proposal attempt.""" + + model_calls: int = Field(ge=0) + retries: int = Field(ge=0) + input_tokens: int | None = Field(default=None, ge=0) + output_tokens: int | None = Field(default=None, ge=0) + wall_time_ms: int | None = Field(default=None, ge=0) + concurrency: int = Field(default=1, ge=1) + validation_failures: int = Field(default=0, ge=0) + + +class Nomination(_ContractModel): + """The proposal's relationship to the existing Candidate path.""" + + status: Literal["not_nominated", "nominated", "rejected"] + reason: str = Field(min_length=1) + candidate_ref: str | None = None + + +class DreamRunRef(_ContractModel): + """Stable reference to the exact Runtime Dream run used for recording.""" + + scope_id: str = Field(min_length=1) + run_id: str = Field(min_length=1) + + +class ProposalAttempt(_ContractModel): + """One realized action; this record never grants Runtime authority.""" + + attempt_id: str = Field(min_length=1) + decision: DecisionRequest + parent_dream_run_ref: DreamRunRef + input_refs: tuple[str, ...] = () + generation: GenerationIdentity + result: AttemptResult + validators: tuple[ValidationResult, ...] = () + cost: AttemptCost + nomination: Nomination + + +class ArtifactEvolutionBundle(_ContractModel): + """Evaluation-owned recording and replay bundle.""" + + schema_version: Literal["powercontext.artifact-evolution/v1"] = Field(alias="schema") + bundle_id: str = Field(min_length=1) + created_at: datetime + workload: Workload + capture_policy: CapturePolicy + requested_decisions: tuple[DecisionRequest, ...] = Field(min_length=1) + requested_trajectories: tuple[TrajectoryRequest, ...] = Field(min_length=1) + attempts: tuple[ProposalAttempt, ...] = () + + @model_validator(mode="after") + def _validate_references(self) -> ArtifactEvolutionBundle: + decisions = {decision.key: decision for decision in self.requested_decisions} + if len(decisions) != len(self.requested_decisions): + raise ValueError("requested_decisions must have unique keys") + decision_ids = {decision.decision_id for decision in self.requested_decisions} + trajectories = {trajectory.trajectory_id: trajectory for trajectory in self.requested_trajectories} + if len(trajectories) != len(self.requested_trajectories): + raise ValueError("requested_trajectories must have unique ids") + for trajectory in self.requested_trajectories: + if any(decision_id not in decision_ids for decision_id in trajectory.decision_ids): + raise ValueError(f"trajectory {trajectory.trajectory_id!r} references an unknown decision") + if any( + decisions[key].trajectory_id != trajectory.trajectory_id + for key in decisions + if decisions[key].decision_id in trajectory.decision_ids + ): + raise ValueError(f"trajectory {trajectory.trajectory_id!r} contains a decision from another trajectory") + seen_attempts: set[tuple[str, str, str, str, int]] = set() + for attempt in self.attempts: + if attempt.decision.key not in decisions: + raise ValueError(f"attempt {attempt.attempt_id!r} is not bound to a requested decision") + if attempt.decision.key in seen_attempts: + raise ValueError(f"duplicate attempt for decision {attempt.decision.decision_id!r}") + seen_attempts.add(attempt.decision.key) + if self.capture_policy.payloads == "none" and attempt.result.output_payload is not None: + raise ValueError("output_payload requires an approved or redacted capture policy") + if attempt.result.output_payload is not None and attempt.result.output_digest != digest_json( + attempt.result.output_payload + ): + raise ValueError(f"attempt {attempt.attempt_id!r} output_digest does not match output_payload") + if ( + self.capture_policy.payloads == "redacted" + and attempt.result.output_payload is not None + and attempt.result.output_payload != _redact_for_validation(attempt.result.output_payload) + ): + raise ValueError("redacted payloads must not contain sensitive values") + return self + + +def _redact_for_validation(value: dict[str, JsonValue]) -> dict[str, JsonValue]: + # Imported lazily to keep the contract module's public imports small. + from powercontext_eval_artifact_evolution.privacy import redact_payload + + return redact_payload(value) diff --git a/evaluation/artifact_evolution/src/powercontext_eval_artifact_evolution/privacy.py b/evaluation/artifact_evolution/src/powercontext_eval_artifact_evolution/privacy.py new file mode 100644 index 000000000..6486df02b --- /dev/null +++ b/evaluation/artifact_evolution/src/powercontext_eval_artifact_evolution/privacy.py @@ -0,0 +1,61 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Small, deterministic privacy and digest helpers for controlled bundles.""" + +from __future__ import annotations + +import hashlib +import json +import re +from collections.abc import Mapping, Sequence +from typing import Any + +_SENSITIVE_KEY = re.compile( + r"(?:prompt|task|evidence|authorization|token|secret|password|credential|api[_-]?key)", + re.IGNORECASE, +) +_SECRET_VALUE = re.compile(r"(?:Bearer\s+\S+|sk-[A-Za-z0-9_-]+|gh[pousr]_[A-Za-z0-9_]+)") +REDACTED = "[REDACTED]" + + +def canonical_json(value: Any) -> bytes: + """Return a stable UTF-8 JSON representation for evaluation digests.""" + + if hasattr(value, "model_dump"): + value = value.model_dump(mode="json", exclude_none=True) + return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8") + + +def digest_json(value: Any) -> str: + """Return a plain lowercase SHA-256 digest without content disclosure.""" + + return hashlib.sha256(canonical_json(value)).hexdigest() + + +def redact_payload(value: Any, *, _sensitive: bool = False) -> Any: + """Recursively redact sensitive keys and recognizable credential strings.""" + + if _sensitive: + return REDACTED + if isinstance(value, Mapping): + return { + str(key): redact_payload(item, _sensitive=bool(_SENSITIVE_KEY.search(str(key)))) + for key, item in value.items() + } + if isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)): + return [redact_payload(item) for item in value] + if isinstance(value, str): + return _SECRET_VALUE.sub(REDACTED, value) + return value diff --git a/evaluation/artifact_evolution/src/powercontext_eval_artifact_evolution/recording.py b/evaluation/artifact_evolution/src/powercontext_eval_artifact_evolution/recording.py new file mode 100644 index 000000000..16535face --- /dev/null +++ b/evaluation/artifact_evolution/src/powercontext_eval_artifact_evolution/recording.py @@ -0,0 +1,236 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Evaluation-owned adapter from a terminal Runtime Dream snapshot.""" + +from __future__ import annotations + +from datetime import datetime +from typing import Literal, cast + +from pydantic import BaseModel, ConfigDict, Field, JsonValue + +from powercontext_eval_artifact_evolution.bundle import record_attempt +from powercontext_eval_artifact_evolution.models import ( + ArtifactEvolutionBundle, + AttemptCost, + AttemptResult, + DecisionRequest, + DreamRunRef, + GenerationIdentity, + Nomination, + ProposalAttempt, + ValidationResult, +) +from powercontext_eval_artifact_evolution.privacy import digest_json + + +class DreamRecordingError(ValueError): + """A Dream snapshot cannot be recorded under the P1 contract.""" + + +class DreamUsageSnapshot(BaseModel): + model_config = ConfigDict(extra="forbid", strict=True) + + model_calls: int = Field(ge=0) + input_tokens: int | None = Field(default=None, ge=0) + output_tokens: int | None = Field(default=None, ge=0) + + +class DreamRunSnapshot(BaseModel): + """Minimal structural view of ``powercontext.builtin.dream.DreamRun``. + + The evaluation project intentionally does not import the Runtime package. + A real DreamRun can be converted with :func:`snapshot_from_dream_run`. + """ + + model_config = ConfigDict(extra="forbid", frozen=True, strict=True) + + scope_id: str = Field(min_length=1) + run_id: str = Field(min_length=1) + operation: Literal["refine_experience"] + status: Literal["queued", "running", "succeeded", "failed"] + outcome: Literal["proposed", "no_change", "needs_evidence"] | None = None + input_manifest: dict[str, JsonValue] | None = None + candidate: dict[str, JsonValue] | None = None + reason: str | None = None + error: str | None = None + started_at: datetime | None = None + completed_at: datetime | None = None + attempt_count: int = Field(default=0, ge=0) + usage: DreamUsageSnapshot + prompt_version: str = Field(min_length=1) + model_config_id: str | None = None + + +def snapshot_from_dream_run(run: object) -> DreamRunSnapshot: + """Read only the stable fields needed from a real or fixture DreamRun.""" + + return DreamRunSnapshot( + scope_id=_required_attribute(run, "scope_id"), + run_id=_required_attribute(run, "run_id"), + operation=_required_attribute(run, "operation"), + status=_required_attribute(run, "status"), + outcome=getattr(run, "outcome", None), + input_manifest=_dump(getattr(run, "input_manifest", None)), + candidate=_dump(getattr(run, "candidate", None)), + reason=getattr(run, "reason", None), + error=getattr(run, "error", None), + started_at=getattr(run, "started_at", None), + completed_at=getattr(run, "completed_at", None), + attempt_count=getattr(run, "attempt_count", 0), + usage=DreamUsageSnapshot.model_validate(_dump(getattr(run, "usage", None)) or {}), + prompt_version=_required_attribute(run, "prompt_version"), + model_config_id=getattr(run, "model_config_id", None), + ) + + +def decision_for_dream_run( + run: DreamRunSnapshot, + *, + action_signature: str, + ordering: int = 0, + decision_id: str | None = None, + trajectory_id: str | None = None, +) -> DecisionRequest: + """Build the exact decision key while computing its observation digest.""" + + if run.input_manifest is None: + raise DreamRecordingError("refine_experience run has no input_manifest") + return DecisionRequest( + decision_id=decision_id or run.run_id, + trajectory_id=trajectory_id or run.scope_id, + observations_digest=digest_json(run.input_manifest), + action_signature=action_signature, + ordering=ordering, + ) + + +def record_dream_attempt( + bundle: ArtifactEvolutionBundle, + run: DreamRunSnapshot | object, + *, + decision: DecisionRequest, + policy_id: str, + generator_id: str, + runtime_version: str, + attempt_id: str | None = None, +) -> ArtifactEvolutionBundle: + """Map one terminal ``refine_experience`` Dream run into an attempt.""" + + snapshot = run if isinstance(run, DreamRunSnapshot) else snapshot_from_dream_run(run) + if snapshot.operation != "refine_experience": + raise DreamRecordingError("only refine_experience Dream runs are supported") + if snapshot.status not in {"succeeded", "failed"}: + raise DreamRecordingError("only terminal Dream runs can be recorded") + if snapshot.input_manifest is None: + raise DreamRecordingError("refine_experience run has no input_manifest") + expected_digest = digest_json(snapshot.input_manifest) + if decision.observations_digest != expected_digest: + raise DreamRecordingError("decision observations_digest does not match input_manifest") + + output_payload: dict[str, JsonValue] | None = None + if snapshot.outcome == "proposed": + if snapshot.candidate is None: + raise DreamRecordingError("proposed Dream run has no candidate reference") + output_payload = { + "candidate_id": snapshot.candidate.get("candidate_id", ""), + "candidate_version": snapshot.candidate.get("version", 0), + } + outcome: Literal["proposed", "no_change", "needs_evidence", "error"] + if snapshot.status == "failed": + outcome = "error" + else: + outcome = snapshot.outcome or "needs_evidence" + candidate_ref = _candidate_ref(snapshot.candidate) + wall_time_ms = None + if snapshot.started_at is not None and snapshot.completed_at is not None: + wall_time_ms = max(0, int((snapshot.completed_at - snapshot.started_at).total_seconds() * 1000)) + attempt = ProposalAttempt( + attempt_id=attempt_id or f"{snapshot.scope_id}:{snapshot.run_id}", + decision=decision, + parent_dream_run_ref=DreamRunRef(scope_id=snapshot.scope_id, run_id=snapshot.run_id), + input_refs=_input_refs(snapshot.input_manifest), + generation=GenerationIdentity( + policy_id=policy_id, + generator_id=generator_id, + model_id=snapshot.model_config_id or "unknown", + prompt_schema_version=snapshot.prompt_version, + runtime_version=runtime_version, + ), + result=AttemptResult( + outcome=outcome, + output_digest=digest_json(output_payload) if output_payload is not None else None, + output_payload=output_payload, + # Runtime error text is intentionally not copied into an evaluation artifact. + error_code="dream_failed" if outcome == "error" else None, + ), + validators=( + ValidationResult( + validator_id="dream-run-terminal", + version="v1", + status="passed" if snapshot.status == "succeeded" else "failed", + details={"source": "DreamRun.status"}, + ), + ), + cost=AttemptCost( + model_calls=snapshot.usage.model_calls, + retries=max(snapshot.attempt_count - snapshot.usage.model_calls, 0), + input_tokens=snapshot.usage.input_tokens, + output_tokens=snapshot.usage.output_tokens, + wall_time_ms=wall_time_ms, + ), + nomination=Nomination( + status="not_nominated", + reason="Dream candidate remains outside Runtime nomination during P1 recording", + candidate_ref=candidate_ref, + ), + ) + return record_attempt(bundle, attempt) + + +def _required_attribute(value: object, name: str): + result = getattr(value, name, None) + if result is None: + raise DreamRecordingError(f"Dream run is missing {name}") + return result + + +def _dump(value: object) -> dict[str, JsonValue] | None: + if value is None: + return None + if isinstance(value, BaseModel): + value = value.model_dump(mode="json", exclude_none=True) + if not isinstance(value, dict): + raise DreamRecordingError("Dream run nested value must be an object") + return cast(dict[str, JsonValue], value) + + +def _candidate_ref(candidate: dict[str, JsonValue] | None) -> str | None: + if candidate is None: + return None + candidate_id = candidate.get("candidate_id") + version = candidate.get("version") + if not isinstance(candidate_id, str) or not isinstance(version, int): + raise DreamRecordingError("candidate reference must include candidate_id and version") + return f"{candidate_id}@{version}" + + +def _input_refs(manifest: dict[str, JsonValue]) -> tuple[str, ...]: + refs: list[str] = [] + for key in ("artifacts", "sources"): + values = manifest.get(key, []) + if isinstance(values, list): + refs.extend(digest_json(value) for value in values) + return tuple(refs) diff --git a/evaluation/artifact_evolution/src/powercontext_eval_artifact_evolution/replay.py b/evaluation/artifact_evolution/src/powercontext_eval_artifact_evolution/replay.py new file mode 100644 index 000000000..8181d3982 --- /dev/null +++ b/evaluation/artifact_evolution/src/powercontext_eval_artifact_evolution/replay.py @@ -0,0 +1,112 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Read-only exact replay over realized Artifact-evolution attempts.""" + +from __future__ import annotations + +from collections.abc import Iterable +from dataclasses import dataclass + +from powercontext_eval_artifact_evolution.models import ArtifactEvolutionBundle, DecisionRequest, ProposalAttempt + + +@dataclass(frozen=True) +class ReplayResult: + """One exact replay lookup result.""" + + status: str + attempt: ProposalAttempt | None = None + reason: str | None = None + + +@dataclass(frozen=True) +class ReplayCoverage: + """Exact replay support counts, retaining unsupported requests in denominators.""" + + requested_decisions: int + supported_decisions: int + unsupported_decision_ids: tuple[str, ...] + requested_trajectories: int + complete_trajectories: int + saved_model_calls: int + saved_retries: int + saved_input_tokens: int | None + saved_output_tokens: int | None + input_tokens_complete: bool + output_tokens_complete: bool + + @property + def decision_ratio(self) -> float: + return self.supported_decisions / self.requested_decisions + + @property + def trajectory_ratio(self) -> float: + return self.complete_trajectories / self.requested_trajectories + + +class ReplayEngine: + """Replay only what the bundle recorded; never call a model or Runtime.""" + + def __init__(self, bundle: ArtifactEvolutionBundle) -> None: + self._bundle = bundle + self._attempts = {attempt.decision.key: attempt for attempt in bundle.attempts} + + def replay(self, request: DecisionRequest) -> ReplayResult: + """Return an exact realized attempt or deterministic ``out_of_support``.""" + + attempt = self._attempts.get(request.key) + if attempt is None: + return ReplayResult( + status="out_of_support", + reason="no realized attempt has the requested trajectory, observations, and action signature", + ) + return ReplayResult(status="replayed", attempt=attempt) + + def coverage(self) -> ReplayCoverage: + decisions = self._bundle.requested_decisions + supported = sum(1 for decision in decisions if decision.key in self._attempts) + unsupported = tuple(decision.decision_id for decision in decisions if decision.key not in self._attempts) + complete_trajectories = 0 + for trajectory in self._bundle.requested_trajectories: + trajectory_decisions = { + decision.key + for decision in decisions + if decision.trajectory_id == trajectory.trajectory_id + and decision.decision_id in trajectory.decision_ids + } + if trajectory_decisions and trajectory_decisions <= self._attempts.keys(): + complete_trajectories += 1 + costs = tuple(attempt.cost for attempt in self._bundle.attempts) + input_tokens = tuple(cost.input_tokens for cost in costs) + output_tokens = tuple(cost.output_tokens for cost in costs) + return ReplayCoverage( + requested_decisions=len(decisions), + supported_decisions=supported, + unsupported_decision_ids=unsupported, + requested_trajectories=len(self._bundle.requested_trajectories), + complete_trajectories=complete_trajectories, + saved_model_calls=sum(cost.model_calls for cost in costs), + saved_retries=sum(cost.retries for cost in costs), + saved_input_tokens=_sum_optional(input_tokens), + saved_output_tokens=_sum_optional(output_tokens), + input_tokens_complete=all(value is not None for value in input_tokens), + output_tokens_complete=all(value is not None for value in output_tokens), + ) + + +def _sum_optional(values: Iterable[int | None]) -> int | None: + entries = tuple(values) + numbers = tuple(value for value in entries if isinstance(value, int)) + return sum(numbers) if numbers else None diff --git a/evaluation/artifact_evolution/tests/artifact_evolution_unit/__init__.py b/evaluation/artifact_evolution/tests/artifact_evolution_unit/__init__.py new file mode 100644 index 000000000..f9f3188da --- /dev/null +++ b/evaluation/artifact_evolution/tests/artifact_evolution_unit/__init__.py @@ -0,0 +1,15 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for the Artifact-evolution evaluation suite.""" diff --git a/evaluation/artifact_evolution/tests/artifact_evolution_unit/test_bundle.py b/evaluation/artifact_evolution/tests/artifact_evolution_unit/test_bundle.py new file mode 100644 index 000000000..aca99ba38 --- /dev/null +++ b/evaluation/artifact_evolution/tests/artifact_evolution_unit/test_bundle.py @@ -0,0 +1,179 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import json +from datetime import UTC, datetime +from pathlib import Path + +import pytest +from pydantic import ValidationError + +from powercontext_eval_artifact_evolution.bundle import ( + BundleIOError, + BundleValidationError, + load_bundle, + record_attempt, + write_bundle, +) +from powercontext_eval_artifact_evolution.models import ( + ArtifactEvolutionBundle, + AttemptCost, + AttemptResult, + Budget, + CapturePolicy, + DecisionRequest, + DreamRunRef, + GenerationIdentity, + Nomination, + ProposalAttempt, + TrajectoryRequest, + ValidationResult, + Workload, +) +from powercontext_eval_artifact_evolution.privacy import digest_json + +DIGEST = "0123456789abcdef" * 4 + + +def decision( + *, + action: str = "refine-experience:v1", + decision_id: str = "decision-1", + trajectory_id: str = "trajectory-1", +) -> DecisionRequest: + return DecisionRequest( + decision_id=decision_id, + trajectory_id=trajectory_id, + observations_digest=DIGEST, + action_signature=action, + ordering=0, + ) + + +def attempt(request: DecisionRequest | None = None, *, payload: dict[str, object] | None = None) -> ProposalAttempt: + return ProposalAttempt( + attempt_id="attempt-1", + decision=decision() if request is None else request, + parent_dream_run_ref=DreamRunRef(scope_id="scope-1", run_id="dream-run-1"), + input_refs=("experience:exp-1", "source:source-1"), + generation=GenerationIdentity( + policy_id="baseline-policy-v1", + generator_id="experience-generator-v1", + model_id="fixture-model", + prompt_schema_version="dream-input-v1", + runtime_version="runtime-fixture-v1", + ), + result=AttemptResult( + outcome="proposed", + output_digest=DIGEST if payload is None else digest_json(payload), + output_payload=payload, + ), + validators=(ValidationResult(validator_id="schema", version="v1", status="passed"),), + cost=AttemptCost(model_calls=1, retries=0, input_tokens=100, output_tokens=20, wall_time_ms=25), + nomination=Nomination(status="not_nominated", reason="baseline recording"), + ) + + +def bundle(*, attempts: tuple[ProposalAttempt, ...] = ()) -> ArtifactEvolutionBundle: + request = decision() + return ArtifactEvolutionBundle( + schema="powercontext.artifact-evolution/v1", + bundle_id="bundle-1", + created_at=datetime(2026, 10, 17, tzinfo=UTC), + workload=Workload( + experiment_id="experiment-1", + scope_digest=DIGEST, + operation="refine_experience", + fixture_digest=DIGEST, + treatment="baseline", + declared_budget=Budget(max_proposal_attempts=1, max_model_calls=2), + ), + capture_policy=CapturePolicy(mode="controlled", payloads="none", retention_class="test"), + requested_decisions=(request,), + requested_trajectories=(TrajectoryRequest(trajectory_id="trajectory-1", decision_ids=(request.decision_id,)),), + attempts=attempts, + ) + + +def test_bundle_round_trips_and_refuses_overwrite(tmp_path: Path) -> None: + path = write_bundle(bundle(attempts=(attempt(),)), tmp_path / "bundle.json") + + loaded = load_bundle(path) + + assert loaded == bundle(attempts=(attempt(),)) + with pytest.raises(BundleIOError, match="refusing to overwrite"): + write_bundle(loaded, path) + + +def test_load_rejects_unsupported_schema_and_unknown_body_fields(tmp_path: Path) -> None: + path = tmp_path / "invalid.json" + payload = bundle(attempts=(attempt(),)).model_dump(mode="json", by_alias=True) + payload["schema"] = "powercontext.artifact-evolution/v2" + path.write_text(json.dumps(payload), encoding="utf-8") + + with pytest.raises(BundleValidationError, match="invalid artifact-evolution bundle"): + load_bundle(path) + + payload["schema"] = "powercontext.artifact-evolution/v1" + payload["attempts"][0]["prompt"] = "must not be retained" + path.write_text(json.dumps(payload), encoding="utf-8") + with pytest.raises(BundleValidationError, match="invalid artifact-evolution bundle"): + load_bundle(path) + + +def test_record_attempt_revalidates_retention_and_references() -> None: + with pytest.raises(ValidationError, match="output_payload requires"): + record_attempt(bundle(), attempt(payload={"redacted": True})) + + approved = bundle().model_copy( + update={ + "capture_policy": CapturePolicy( + mode="controlled", payloads="approved", retention_class="test", approval_ref="review-1" + ) + } + ) + recorded = record_attempt(approved, attempt(payload={"safe": "keep"})) + assert recorded.attempts[0].result.output_payload == {"safe": "keep"} + + +def test_redacted_capture_removes_sensitive_keys_and_credentials(tmp_path: Path) -> None: + redacted = bundle().model_copy( + update={"capture_policy": CapturePolicy(mode="controlled", payloads="redacted", retention_class="test")} + ) + recorded = record_attempt( + redacted, + attempt( + payload={ + "prompt": "SECRET TASK BODY", + "authorization": "Bearer sk-live-123", + "safe": "keep", + } + ), + ) + + payload = recorded.attempts[0].result.output_payload + assert payload == {"prompt": "[REDACTED]", "authorization": "[REDACTED]", "safe": "keep"} + assert "SECRET TASK BODY" not in json.dumps(payload) + assert "sk-live-123" not in json.dumps(payload) + persisted = write_bundle(recorded, tmp_path / "redacted.json").read_text(encoding="utf-8") + assert "SECRET TASK BODY" not in persisted + assert "sk-live-123" not in persisted + + +def test_attempt_must_belong_to_a_requested_decision() -> None: + other = decision(decision_id="other") + with pytest.raises(ValidationError, match="not bound to a requested decision"): + record_attempt(bundle(), attempt(other)) diff --git a/evaluation/artifact_evolution/tests/artifact_evolution_unit/test_recording.py b/evaluation/artifact_evolution/tests/artifact_evolution_unit/test_recording.py new file mode 100644 index 000000000..c631fe767 --- /dev/null +++ b/evaluation/artifact_evolution/tests/artifact_evolution_unit/test_recording.py @@ -0,0 +1,142 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from datetime import UTC, datetime +from pathlib import Path + +from typer.testing import CliRunner + +from powercontext_eval_artifact_evolution.bundle import load_bundle, write_bundle +from powercontext_eval_artifact_evolution.cli import app +from powercontext_eval_artifact_evolution.models import CapturePolicy, TrajectoryRequest +from powercontext_eval_artifact_evolution.privacy import digest_json +from powercontext_eval_artifact_evolution.recording import ( + DreamRunSnapshot, + DreamUsageSnapshot, + decision_for_dream_run, + record_dream_attempt, +) +from powercontext_eval_artifact_evolution.replay import ReplayEngine + +from .test_bundle import bundle + + +def dream_run() -> DreamRunSnapshot: + return DreamRunSnapshot( + scope_id="scope-1", + run_id="dream-run-1", + operation="refine_experience", + status="succeeded", + outcome="proposed", + input_manifest={"projection_digest": "manifest-1", "nodes": [{"evidence_id": "e-1"}]}, + candidate={"kind": "artifact", "candidate_id": "candidate-1", "version": 2}, + started_at=datetime(2026, 10, 17, 12, 0, tzinfo=UTC), + completed_at=datetime(2026, 10, 17, 12, 0, 1, 500_000, tzinfo=UTC), + attempt_count=2, + usage=DreamUsageSnapshot(model_calls=1, input_tokens=100, output_tokens=20), + prompt_version="powercontext.dream.v1.4", + model_config_id="model-1", + ) + + +def test_controlled_dream_recording_round_trips_into_exact_replay(tmp_path: Path) -> None: + run = dream_run() + decision = decision_for_dream_run(run, action_signature="refine-experience:v1") + base = bundle().model_copy( + update={ + "capture_policy": CapturePolicy(mode="controlled", payloads="redacted", retention_class="test"), + "requested_decisions": (decision,), + "requested_trajectories": ( + TrajectoryRequest(trajectory_id=decision.trajectory_id, decision_ids=(decision.decision_id,)), + ), + } + ) + + recorded = record_dream_attempt( + base, + run, + decision=decision, + policy_id="baseline-policy-v1", + generator_id="dream.refine_experience", + runtime_version="runtime-v1", + ) + path = write_bundle(recorded, tmp_path / "recorded.json") + loaded = load_bundle(path) + replayed = ReplayEngine(loaded).replay(decision) + + assert replayed.status == "replayed" + assert replayed.attempt is not None + assert replayed.attempt.parent_dream_run_ref.run_id == "dream-run-1" + assert replayed.attempt.cost.retries == 1 + assert replayed.attempt.cost.wall_time_ms == 1500 + assert replayed.attempt.result.output_digest == digest_json(replayed.attempt.result.output_payload) + + +def test_recorder_rejects_non_terminal_or_wrong_operation() -> None: + run = dream_run() + decision = decision_for_dream_run(run, action_signature="refine-experience:v1") + running = run.model_copy(update={"status": "running"}) + + import pytest + + with pytest.raises(ValueError, match="terminal"): + record_dream_attempt( + bundle(), + running, + decision=decision, + policy_id="p", + generator_id="g", + runtime_version="r", + ) + + +def test_cli_records_a_controlled_dream_snapshot(tmp_path: Path) -> None: + run = dream_run() + decision = decision_for_dream_run(run, action_signature="refine-experience:v1") + base = bundle().model_copy( + update={ + "capture_policy": CapturePolicy(mode="controlled", payloads="redacted", retention_class="test"), + "requested_decisions": (decision,), + "requested_trajectories": ( + TrajectoryRequest(trajectory_id=decision.trajectory_id, decision_ids=(decision.decision_id,)), + ), + } + ) + source = write_bundle(base, tmp_path / "base.json") + dream_path = tmp_path / "dream.json" + dream_path.write_text(run.model_dump_json(), encoding="utf-8") + output = tmp_path / "recorded.json" + + result = CliRunner().invoke( + app, + [ + "record", + str(source), + str(dream_path), + str(output), + "--action-signature", + "refine-experience:v1", + "--policy-id", + "baseline-policy-v1", + "--generator-id", + "dream.refine_experience", + "--runtime-version", + "runtime-v1", + ], + ) + + assert result.exit_code == 0, result.output + assert load_bundle(output).attempts[0].parent_dream_run_ref.run_id == "dream-run-1" diff --git a/evaluation/artifact_evolution/tests/artifact_evolution_unit/test_replay.py b/evaluation/artifact_evolution/tests/artifact_evolution_unit/test_replay.py new file mode 100644 index 000000000..2f222fa60 --- /dev/null +++ b/evaluation/artifact_evolution/tests/artifact_evolution_unit/test_replay.py @@ -0,0 +1,190 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from pathlib import Path + +import pytest +from pydantic import ValidationError +from typer.testing import CliRunner + +from powercontext_eval_artifact_evolution.bundle import write_bundle +from powercontext_eval_artifact_evolution.cli import app +from powercontext_eval_artifact_evolution.models import ArtifactEvolutionBundle, AttemptCost, TrajectoryRequest +from powercontext_eval_artifact_evolution.replay import ReplayEngine + +from .test_bundle import DIGEST, attempt, bundle, decision + + +def test_replay_returns_saved_result_and_cost_without_reexecution() -> None: + recorded = attempt() + engine = ReplayEngine(bundle(attempts=(recorded,))) + + result = engine.replay(decision()) + + assert result.status == "replayed" + assert result.attempt == recorded + coverage = engine.coverage() + assert coverage.decision_ratio == 1.0 + assert coverage.trajectory_ratio == 1.0 + assert coverage.saved_model_calls == 1 + assert coverage.saved_retries == 0 + assert coverage.saved_input_tokens == 100 + assert coverage.saved_output_tokens == 20 + assert coverage.input_tokens_complete is True + assert coverage.output_tokens_complete is True + + +def test_replay_is_read_only_and_does_not_consume_future_outcomes() -> None: + state = {"model_calls": 0, "candidate_count": 1, "review_state": "pending"} + engine = ReplayEngine(bundle(attempts=(attempt(),))) + before = state.copy() + + result = engine.replay(decision()) + + assert state == before + assert result.status == "replayed" + assert result.attempt is not None + assert "review" not in result.attempt.model_dump(mode="python") + assert "revision" not in result.attempt.model_dump(mode="python") + + # Downstream state is deliberately not an input to the exact replay key. + future_state = {"review": "approved", "revision": 7, "selected": True} + assert engine.replay(decision()).attempt == result.attempt + assert future_state == {"review": "approved", "revision": 7, "selected": True} + + +def test_attempt_rejects_downstream_result_fields() -> None: + raw = attempt().model_dump(mode="python") + raw["review"] = {"status": "approved"} + with pytest.raises(ValidationError, match="Extra inputs are not permitted"): + type(attempt()).model_validate(raw) + + +def test_replay_is_out_of_support_for_an_unrecorded_action() -> None: + result = ReplayEngine(bundle(attempts=(attempt(),))).replay(decision(action="refine-experience:unrecorded")) + + assert result.status == "out_of_support" + assert result.attempt is None + assert result.reason is not None + + +def test_coverage_keeps_unsupported_decisions_and_trajectories_in_denominator() -> None: + second = decision(decision_id="decision-2", action="refine-experience:second", trajectory_id="trajectory-2") + expanded = bundle().model_copy( + update={ + "requested_decisions": (decision(), second), + "requested_trajectories": ( + TrajectoryRequest(trajectory_id="trajectory-1", decision_ids=("decision-1",)), + TrajectoryRequest(trajectory_id="trajectory-2", decision_ids=("decision-2",)), + ), + "attempts": (attempt(),), + } + ) + # Validate the copy through the public model boundary before replaying it. + validated = ArtifactEvolutionBundle.model_validate(expanded.model_dump(mode="python", by_alias=True)) + coverage = ReplayEngine(validated).coverage() + + assert coverage.requested_decisions == 2 + assert coverage.supported_decisions == 1 + assert coverage.decision_ratio == 0.5 + assert coverage.requested_trajectories == 2 + assert coverage.complete_trajectories == 1 + assert coverage.trajectory_ratio == 0.5 + assert coverage.unsupported_decision_ids == ("decision-2",) + + +def test_coverage_preserves_known_partial_token_costs() -> None: + second = decision(decision_id="decision-2", action="refine-experience:second", trajectory_id="trajectory-2") + partial = attempt(second).model_copy( + update={ + "cost": AttemptCost(model_calls=1, retries=0, input_tokens=None, output_tokens=10), + } + ) + expanded = bundle().model_copy( + update={ + "requested_decisions": (decision(), second), + "requested_trajectories": ( + TrajectoryRequest(trajectory_id="trajectory-1", decision_ids=("decision-1",)), + TrajectoryRequest(trajectory_id="trajectory-2", decision_ids=("decision-2",)), + ), + "attempts": (attempt(), partial), + } + ) + coverage = ReplayEngine( + ArtifactEvolutionBundle.model_validate(expanded.model_dump(mode="python", by_alias=True)) + ).coverage() + + assert coverage.saved_input_tokens == 100 + assert coverage.input_tokens_complete is False + assert coverage.saved_output_tokens == 30 + assert coverage.output_tokens_complete is True + + +def test_cli_validate_replay_and_coverage_use_saved_bundle_only(tmp_path: Path) -> None: + path = write_bundle(bundle(attempts=(attempt(),)), tmp_path / "bundle.json") + runner = CliRunner() + + validated = runner.invoke(app, ["validate", str(path)]) + replayed = runner.invoke( + app, + [ + "replay", + str(path), + "--decision-id", + "decision-1", + "--trajectory-id", + "trajectory-1", + "--observations-digest", + DIGEST, + "--action-signature", + "refine-experience:v1", + "--ordering", + "0", + ], + ) + coverage = runner.invoke(app, ["coverage", str(path)]) + + assert validated.exit_code == 0, validated.stdout + assert '"recorded_attempts": 1' in validated.stdout + assert replayed.exit_code == 0, replayed.stdout + assert '"status": "replayed"' in replayed.stdout + assert coverage.exit_code == 0, coverage.stdout + assert '"decision_ratio": 1.0' in coverage.stdout + + +def test_cli_rejects_malformed_decision_request_without_traceback(tmp_path: Path) -> None: + path = write_bundle(bundle(attempts=(attempt(),)), tmp_path / "bundle.json") + result = CliRunner().invoke( + app, + [ + "replay", + str(path), + "--decision-id", + "decision-1", + "--trajectory-id", + "trajectory-1", + "--observations-digest", + "abc", + "--action-signature", + "refine-experience:v1", + "--ordering", + "0", + ], + ) + + assert result.exit_code == 1 + assert "invalid decision request" in result.output + assert "Traceback" not in result.output diff --git a/evaluation/pyproject.toml b/evaluation/pyproject.toml index a1d4e53f4..caacb10a5 100644 --- a/evaluation/pyproject.toml +++ b/evaluation/pyproject.toml @@ -31,6 +31,7 @@ dependencies = [ swebench-pro = "powercontext_eval_swebench_pro.cli:main" longmemeval-v2 = "powercontext_eval_longmemeval_v2.cli:main" work-continuity = "powercontext_eval_work_continuity.cli:main" +artifact-evolution = "powercontext_eval_artifact_evolution.cli:main" [dependency-groups] dev = [ @@ -46,10 +47,11 @@ packages = [ "coding/swebench_pro/src/powercontext_eval_swebench_pro", "coding/work_continuity/src/powercontext_eval_work_continuity", "memory/longmemeval_v2/src/powercontext_eval_longmemeval_v2", + "artifact_evolution/src/powercontext_eval_artifact_evolution", ] [tool.hatch.build.targets.sdist] -include = ["/coding/swebench_pro", "/coding/work_continuity", "/memory/longmemeval_v2", "/README.md", "/pyproject.toml", "/uv.lock"] +include = ["/artifact_evolution", "/coding/swebench_pro", "/coding/work_continuity", "/memory/longmemeval_v2", "/README.md", "/pyproject.toml", "/uv.lock"] exclude = ["**/.venv", "**/node_modules", "**/dist", "**/__pycache__", "**/.ruff_cache", "**/.pytest_cache", "**/*.tsbuildinfo", "**/test-results", "**/playwright-report"] [tool.ty.environment] @@ -57,10 +59,10 @@ python = "./.venv" python-version = "3.11" [tool.ty.src] -include = ["coding/swebench_pro/src", "coding/work_continuity/src", "memory/longmemeval_v2/src"] +include = ["artifact_evolution/src", "coding/swebench_pro/src", "coding/work_continuity/src", "memory/longmemeval_v2/src"] [tool.pytest.ini_options] -testpaths = ["coding/swebench_pro/tests", "coding/work_continuity/tests", "memory/longmemeval_v2/tests"] +testpaths = ["artifact_evolution/tests", "coding/swebench_pro/tests", "coding/work_continuity/tests", "memory/longmemeval_v2/tests"] addopts = ["--import-mode=importlib"] markers = [ "live: tests that require external services", diff --git a/pyproject.toml b/pyproject.toml index 9d08ce372..2287f9023 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -200,6 +200,7 @@ exclude = [ "tests/langgraph_adapter", "tests/e2e/test_langgraph_chain.py", "evaluation/coding", + "evaluation/artifact_evolution", "evaluation/memory/longmemeval_v2", "evaluation/skills", "evaluation/zcode_guidance",