diff --git a/.github/workflows/aeg-arm-execution-substrate.yml b/.github/workflows/aeg-arm-execution-substrate.yml new file mode 100644 index 0000000..fe63b8d --- /dev/null +++ b/.github/workflows/aeg-arm-execution-substrate.yml @@ -0,0 +1,220 @@ +name: AEG Arm Execution Substrate + +on: + workflow_dispatch: + inputs: + operation: + description: Run the canary or, after separate authorization, the frozen S1 matrix + required: true + type: choice + default: canary + options: + - canary + - execute-s1 + confirmation: + description: Required only for execute-s1 + required: false + type: string + pull_request: + paths: + - ".github/workflows/aeg-arm-execution-substrate.yml" + - "infrastructure/aeg-arm-execution-substrate/**" + - "experiments/situated-experience-benchmark-v1/execution/PROTOCOL-DEVIATION-ARM-SUBSTRATE.md" + +permissions: + contents: read + +concurrency: + group: aeg-arm-substrate-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: false + +env: + AEG_SUBSTRATE_IMAGE: aeg-arm-runner:python3.12.11-slim-bookworm-v1 + AEG_SUBSTRATE_DIR: infrastructure/aeg-arm-execution-substrate + +jobs: + hosted-canary: + name: Hosted substrate canary + runs-on: ubuntu-24.04 + timeout-minutes: 25 + steps: + - name: Check out trusted controller source without credentials + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + with: + persist-credentials: false + + - name: Validate frozen inputs and substrate policy + run: | + python3 "$AEG_SUBSTRATE_DIR/controller.py" validate + python3 -m unittest discover -s "$AEG_SUBSTRATE_DIR/tests" -p 'test_*.py' -v + + - name: Build the digest-pinned repair image + run: docker build --pull --tag "$AEG_SUBSTRATE_IMAGE" "$AEG_SUBSTRATE_DIR" + + - name: Revalidate every S1 fixture in the pinned image + run: | + python3 "$AEG_SUBSTRATE_DIR/controller.py" revalidate-fixtures \ + --image "$AEG_SUBSTRATE_IMAGE" \ + --output "$RUNNER_TEMP/aeg-fixture-revalidation.json" + + - name: Run complete hosted adversarial canary + id: canary + continue-on-error: true + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + AEG_RAW_OUTPUT_CERT_PEM: ${{ secrets.AEG_RAW_OUTPUT_CERT_PEM }} + run: | + python3 "$AEG_SUBSTRATE_DIR/controller.py" canary \ + --image "$AEG_SUBSTRATE_IMAGE" \ + --fixture-record "$RUNNER_TEMP/aeg-fixture-revalidation.json" \ + --output "$RUNNER_TEMP/aeg-substrate-canary.json" \ + --encrypted-raw-output "$RUNNER_TEMP/aeg-substrate-canary-raw.p7m" + + - name: Upload sanitized canary and fixture evidence + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: aeg-substrate-canary-sanitized-${{ github.run_id }} + path: | + ${{ runner.temp }}/aeg-substrate-canary.json + ${{ runner.temp }}/aeg-fixture-revalidation.json + if-no-files-found: error + retention-days: 30 + + - name: Upload encrypted raw canary output + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: aeg-substrate-canary-encrypted-${{ github.run_id }} + path: ${{ runner.temp }}/aeg-substrate-canary-raw.p7m + if-no-files-found: ignore + retention-days: 30 + + - name: Enforce hosted readiness gate + if: always() + env: + CANARY_OUTCOME: ${{ steps.canary.outcome }} + run: test "$CANARY_OUTCOME" = success + + arm-matrix: + name: Frozen S1 arm ${{ matrix.sequence }} ${{ matrix.arm_id }} + if: >- + github.event_name == 'workflow_dispatch' && + inputs.operation == 'execute-s1' && + inputs.confirmation == 'EXECUTE_FROZEN_S1_95ce8de8_12_ARMS' + needs: hosted-canary + runs-on: ubuntu-24.04 + timeout-minutes: 30 + strategy: + fail-fast: false + max-parallel: 1 + matrix: + include: + - sequence: 1 + arm_id: s1-01-scrapy-cookiejar--r01--aeg-assisted + pair: s1-01-scrapy-cookiejar + replicate: 1 + mode: aeg-assisted + - sequence: 2 + arm_id: s1-01-scrapy-cookiejar--r01--control + pair: s1-01-scrapy-cookiejar + replicate: 1 + mode: control + - sequence: 3 + arm_id: s1-01-scrapy-cookiejar--r02--aeg-assisted + pair: s1-01-scrapy-cookiejar + replicate: 2 + mode: aeg-assisted + - sequence: 4 + arm_id: s1-01-scrapy-cookiejar--r02--control + pair: s1-01-scrapy-cookiejar + replicate: 2 + mode: control + - sequence: 5 + arm_id: s1-01-scrapy-cookiejar--r03--aeg-assisted + pair: s1-01-scrapy-cookiejar + replicate: 3 + mode: aeg-assisted + - sequence: 6 + arm_id: s1-01-scrapy-cookiejar--r03--control + pair: s1-01-scrapy-cookiejar + replicate: 3 + mode: control + - sequence: 7 + arm_id: s1-02-fastapi-pydantic--r01--control + pair: s1-02-fastapi-pydantic + replicate: 1 + mode: control + - sequence: 8 + arm_id: s1-02-fastapi-pydantic--r01--aeg-assisted + pair: s1-02-fastapi-pydantic + replicate: 1 + mode: aeg-assisted + - sequence: 9 + arm_id: s1-02-fastapi-pydantic--r02--control + pair: s1-02-fastapi-pydantic + replicate: 2 + mode: control + - sequence: 10 + arm_id: s1-02-fastapi-pydantic--r02--aeg-assisted + pair: s1-02-fastapi-pydantic + replicate: 2 + mode: aeg-assisted + - sequence: 11 + arm_id: s1-02-fastapi-pydantic--r03--control + pair: s1-02-fastapi-pydantic + replicate: 3 + mode: control + - sequence: 12 + arm_id: s1-02-fastapi-pydantic--r03--aeg-assisted + pair: s1-02-fastapi-pydantic + replicate: 3 + mode: aeg-assisted + steps: + - name: Check out trusted controller source without credentials + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + with: + persist-credentials: false + + - name: Validate frozen inputs and matrix coordinate + run: python3 "$AEG_SUBSTRATE_DIR/controller.py" validate + + - name: Build the digest-pinned repair image + run: docker build --pull --tag "$AEG_SUBSTRATE_IMAGE" "$AEG_SUBSTRATE_DIR" + + - name: Package exactly one frozen arm + run: | + python3 experiments/situated-experience-benchmark-v1/run_benchmark.py package-arm \ + --pair "${{ matrix.pair }}" \ + --replicate "${{ matrix.replicate }}" \ + --mode "${{ matrix.mode }}" \ + --output "$RUNNER_TEMP/one-arm" + + - name: Execute repair, terminate it, then evaluate in a separate container + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + AEG_RAW_OUTPUT_CERT_PEM: ${{ secrets.AEG_RAW_OUTPUT_CERT_PEM }} + run: | + python3 "$AEG_SUBSTRATE_DIR/controller.py" execute-arm \ + --image "$AEG_SUBSTRATE_IMAGE" \ + --bundle "$RUNNER_TEMP/one-arm" \ + --arm-id "${{ matrix.arm_id }}" \ + --sequence "${{ matrix.sequence }}" \ + --sanitized-output "$RUNNER_TEMP/sanitized-arm-result.json" \ + --encrypted-raw-output "$RUNNER_TEMP/encrypted-arm-raw.p7m" + + - name: Upload sanitized arm metrics + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: seb-s1-sanitized-${{ matrix.sequence }}-${{ matrix.arm_id }} + path: ${{ runner.temp }}/sanitized-arm-result.json + if-no-files-found: error + retention-days: 30 + + - name: Upload encrypted raw transcript and patch + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: seb-s1-encrypted-${{ matrix.sequence }}-${{ matrix.arm_id }} + path: ${{ runner.temp }}/encrypted-arm-raw.p7m + if-no-files-found: error + retention-days: 30 diff --git a/.github/workflows/repair-lab.yml b/.github/workflows/repair-lab.yml index 95420a0..0e2a14e 100644 --- a/.github/workflows/repair-lab.yml +++ b/.github/workflows/repair-lab.yml @@ -6,6 +6,8 @@ on: - "experiences/**" - "experiments/public-repair-lab/**" - "experiments/natural-transfer-benchmark/**" + - "experiments/situated-experience-benchmark-v1/**" + - "experiments/v0.1.6-product-proof/**" - "integrations/vscode/**" - "scripts/**" - "references/trace_schema.md" @@ -16,12 +18,16 @@ on: - ".github/workflows/natural-transfer-isolation.yml" - ".github/workflows/model-cost-feasibility.yml" - "experiments/model-cost-feasibility/**" + - "infrastructure/aeg-arm-execution-substrate/**" + - ".github/workflows/aeg-arm-execution-substrate.yml" push: branches: [main] paths: - "experiences/**" - "experiments/public-repair-lab/**" - "experiments/natural-transfer-benchmark/**" + - "experiments/situated-experience-benchmark-v1/**" + - "experiments/v0.1.6-product-proof/**" - "integrations/vscode/**" - "scripts/**" - "references/trace_schema.md" @@ -32,6 +38,8 @@ on: - ".github/workflows/natural-transfer-isolation.yml" - ".github/workflows/model-cost-feasibility.yml" - "experiments/model-cost-feasibility/**" + - "infrastructure/aeg-arm-execution-substrate/**" + - ".github/workflows/aeg-arm-execution-substrate.yml" workflow_dispatch: permissions: @@ -80,6 +88,7 @@ jobs: experiments/natural-transfer-benchmark/manifest.json \ >/dev/null python3 experiments/natural-transfer-benchmark/run_benchmark.py validate + python3 experiments/situated-experience-benchmark-v1/run_benchmark.py validate - name: Test retrieval, validation, telemetry, and task preparation run: | @@ -94,6 +103,10 @@ jobs: python3 experiments/natural-transfer-benchmark/run_benchmark.py self-test python3 experiments/natural-transfer-benchmark/test_run_benchmark.py python3 experiments/natural-transfer-benchmark/test_isolation_controller.py + python3 experiments/situated-experience-benchmark-v1/run_benchmark.py preflight + python3 experiments/situated-experience-benchmark-v1/test_benchmark.py + python3 infrastructure/aeg-arm-execution-substrate/controller.py validate + python3 -m unittest discover -s infrastructure/aeg-arm-execution-substrate/tests -p 'test_*.py' -v - name: Install extension dependencies working-directory: integrations/vscode diff --git a/README.md b/README.md index 3df3ea7..1bd0add 100644 --- a/README.md +++ b/README.md @@ -3,11 +3,12 @@ Agent Experience Graph helps coding agents retrieve verified debugging experience instead of solving every problem from scratch. -The v0.1.5 developer preview turns a task or error into an explainable match -from a small verified-only public library, then produces a compact recovery -capsule for the coding agent. The capsule preserves lessons, failed approaches, -constraints, limitations, and public provenance. It is guidance to validate, -not a guaranteed answer. +The v0.1.6 product-proof release gives the VS Code extension one honest path: +a task or error becomes an explainable verified match or explicit abstention; +an above-threshold match exposes evidence and limitations before a guarded +capsule is copied; the user then records an objective validation outcome and +local usefulness rating. It retrieves guidance and does not automatically +solve, send, or run the task. Think of it like a shared memory of successful work patterns. When an agent starts a new task, it can look at previous tasks, see which parts were similar, and learn which tools, skills, and approaches helped before. @@ -28,18 +29,35 @@ Open the investor- and partner-friendly living pitch: The pitch covers the vision, problem, architecture, initial product, early progress, business model, roadmap, founder, and current ask. -## Try a Verified Experience in VS Code +## v0.1.6 VS Code quick start -Install [AEG from the Visual Studio Marketplace](https://marketplace.visualstudio.com/items?itemName=AgentExperienceGraph.agent-experience-graph), -then use this three-step quick start: +The v0.1.6 clean-profile founder usability gate passed on 2026-08-14 using the +local VSIX with SHA-256 +`18ef493b9290e28832e54527d7fb92624387a17d749ec228b60087c3b6917224`. +On first activation in a normal workspace, AEG opens the founder walkthrough +once. If opening is skipped or fails, the persistent **AEG: Start here** +status-bar action is the primary entry without README or Command Palette +knowledge. The walkthrough also remains available from the AEG sidebar. -1. Run **AEG: Try a Verified Experience** and describe a task or select an error. -2. Inspect why a verified record matched, then copy its guarded capsule before - the coding agent begins. -3. Validate the repair and rate the retrieval locally. +Then follow this quick start: -For zero-cold-start onboarding, run **AEG: Open Verified Experience -Challenge**. It uses a bundled synthetic transfer fixture and openly reports +1. Run **AEG: Start with Verified Experience** and describe a task or select an error. +2. Inspect the score, matching phrases, provenance, constraints, and limitations—or accept **No relevant verified experience** as a correct outcome. +3. Copy an above-threshold guarded capsule. Open VS Code Chat, paste it into the chat input with the original task, and press Enter. +4. Run focused and regression checks, record the observed outcome, then rate the selected experience locally. + +The sidebar shows the honest boundary up front: **2 verified records · 2 task +families**. Playwright diagnosis, Repair Lab, skill discovery, the synthetic +challenge, and legacy commands remain available under **Advanced**. + +The founder pass validates usability and discoverability only. The product-proof +experiment remains prepared, not frozen, with 0/3 arms executed; it supports no +claim of better repair success, speed, cost, adoption, product-market fit, or +generalization. See +[`experiments/v0.1.6-product-proof/UX-ACCEPTANCE.md`](experiments/v0.1.6-product-proof/UX-ACCEPTANCE.md). + +For zero-cold-start onboarding, use the walkthrough's bundled guided task. The +Advanced **Open Bundled Transfer Challenge** command uses a synthetic fixture and reports that its prior controlled pair found no repair-path or outcome improvement and higher assisted token and wall-time cost. See [`experiments/verified-experience-challenge/`](experiments/verified-experience-challenge/). @@ -67,9 +85,10 @@ higher assisted token and wall-time cost. See ## 60-Second Demo -Run **AEG: Open Verified Experience Challenge** from the VS Code command -palette. The demo shows task entry, verified-only retrieval, weighted match -evidence, a compact guarded capsule, and local usefulness feedback. +Select the visible **AEG: Start here** status-bar action or use the bundled task +in the walkthrough. The demo shows task entry, +verified-only retrieval, weighted match evidence, a compact guarded capsule, +explicit paste instructions, validation, and local usefulness feedback. Use [`docs/60-second-demo.md`](docs/60-second-demo.md) for a short meeting talk track. @@ -85,7 +104,8 @@ Agent Experience Graph gives agents a way to ask: - Which tools failed or wasted time? - What should I watch out for? -This can make agents more reliable, faster to start, and easier to improve over time. +Whether this makes agents more reliable or efficient is an open question that +requires controlled evidence beyond the current two-record library. ## A Non-Technical Example @@ -112,8 +132,8 @@ It is intentionally runtime-neutral: - `scripts/recommend_traces.py` ranks similar traces and recommends reusable skills/tools. - `references/trace_schema.md` defines the trace data contract. - `experiences/verified.json` stores sanitized, executed, and objectively verified shared experiences. -- `integrations/vscode/` exposes the verified-experience challenge and the - existing Playwright workflow. +- `integrations/vscode/` exposes the v0.1.6 verified-experience golden path and + keeps prior Playwright, Repair Lab, and skill tools under Advanced. - `experiments/verified-experience-challenge/` supplies a transparent bundled transfer demo. - `experiments/public-repair-lab/` runs the first baseline-versus-AEG public bug repair experiment. @@ -143,6 +163,25 @@ median and 732 fewer non-cached tokens, while wall time regressed by 18.2 second This is a bounded tool-cycle/cost signal on one task family, not a general speed or success-rate claim. See `experiments/public-repair-lab/RESULTS.md`. +## Situated Experience Benchmark v1 + +`experiments/situated-experience-benchmark-v1/` stages a broader, ordered test +of whether AEG helps when repair depends on version state, execution environment, +historical failures, cross-module consequences, multi-agent handoffs, and +experience applicability. Its six families run from dependency migration (S1) +through experience invalidation under environment drift (S6). + +Only S1 is implemented. Exactly two natural public source-transfer pairs, +Scrapy/Python CookieJar and FastAPI/Pydantic field representations, are frozen +with offline fixtures, hidden evaluators, deterministic three-replicate arm +orders, common measurement rules, and fail-closed isolation/leakage preflights. +No benchmark arm has run, and S2-S6 remain screening rules only. + +```bash +python3 experiments/situated-experience-benchmark-v1/run_benchmark.py validate +python3 experiments/situated-experience-benchmark-v1/run_benchmark.py preflight +``` + In plain English, this repository contains: - a guide that tells an agent how to use prior experience @@ -183,10 +222,10 @@ This project may be useful for: Agent Experience Graph is designed around sanitized traces, not raw logs. -The v0.1.5 extension bundles its public verified library and performs retrieval -locally. It does not upload task text, code, logs, recovery capsules, receipts, -or usefulness ratings. Local ratings are stored under `.aeg/`; review or ignore -that directory before committing it. +The v0.1.6 extension bundles its public verified library and performs retrieval +locally. It does not upload task text, code, prompts, logs, recovery capsules, +receipts, ratings, or private data. Local validation outcomes and ratings are +stored under `.aeg/`; review or ignore that directory before committing it. That means shared traces should not contain: diff --git a/docs/60-second-demo.md b/docs/60-second-demo.md index 671492f..bfe29c8 100644 --- a/docs/60-second-demo.md +++ b/docs/60-second-demo.md @@ -1,54 +1,43 @@ -# 60-Second Demo Talk Track +# 60-second v0.1.6 demo ## Goal -Show one memorable moment: +Show one honest loop: -> A developer gives AEG a debugging task and retrieves a verified, explainable -> recovery capsule before the coding agent starts from scratch. +> Task or error → verified match or abstention → inspect evidence → guarded handoff → validate → local feedback. -This demo is intentionally lightweight. It is not meant to prove the final architecture. It is meant to make the product abstraction easy to understand in a short meeting. +This is an interaction demo. It does not prove that retrieval improves correctness, success, speed, cost, adoption, or generalization. -## Demo Script +## Talk track -Install AEG v0.1.5, open the repository in VS Code, and run **AEG: Open -Verified Experience Challenge**. +Install AEG v0.1.6, open a test workspace in VS Code, and run **AEG: Start with Verified Experience**. -Say: +Point out the sidebar first: -> This public wrapper still uses a stale resource after ownership moved behind a -> protocol layer. Before the coding agent starts, AEG searches only its bundled -> verified records. +> There is one primary action. Coverage is visible: two verified records in two narrow task families. Playwright, Repair Lab, skill discovery, the synthetic challenge, and legacy commands remain under Advanced. -Select the TR-04 result and point to **Why this matched**. +Use this task: -Then say: +```text +Keepalive control fails after active stream ownership moved behind a protocol object; repair the public wrapper so it delegates through the protocol without using its stale socket field. +``` -> AEG exposes the exact fields and weighted lexical evidence behind the match. -> The experience records the original failed client-side approach, the recovery -> principle, constraints, limitations, and public provenance. +Select the TR-04 result and say: -Click **Copy capsule**. +> AEG exposes the exact matching phrases and weighted lexical score, then shows the objectively checked source outcome, public provenance, constraints, and limitations. Verified describes the source record; it does not guarantee this task. -> The capsule is guidance, not an answer guarantee. It tells the agent to inspect -> the local code, reproduce the failure, and validate the patch. +Select **Copy capsule** and point to the instructions: -Point to the rating buttons: +> AEG copies guarded guidance and tells me exactly where to paste it. It does not call a private chat API, submit a prompt, or run an agent. -> After validation, the developer records whether the experience was helpful, -> partially helpful, irrelevant, or harmful. That feedback stays in `.aeg/`. +Point to the disabled-then-enabled validation and rating steps: -## Core Message +> I must record an observed focused/regression-check outcome before I can rate this experience. The query, selected experience, validation result, and rating stay together in `.aeg/` and are not uploaded. -AEG helps coding agents retrieve verified debugging experience instead of -solving every problem from scratch. The current evidence is deliberately -narrow: the prior pair for this synthetic challenge found the same successful -repair in both arms and higher assisted token and wall-time cost. +If time permits, start again with `Change the website navigation background from white to blue and increase the logo size.` Show **No relevant verified experience**: -## Questions to Ask +> Abstention is a correct outcome. AEG shows the score and threshold, discloses current coverage, and injects no generic fallback. -- What is the right abstraction for reusable agent skills? -- Are failure patterns more reusable than successful workflows? -- Should skills be prompts, tools, workflows, policies, or all of the above? -- Where would this fit in the current agent developer ecosystem? -- What would make this useful for real developers rather than just an interesting demo? +## Evidence boundary + +The bundled task is a synthetic transfer demonstration. Its prior controlled pair produced the same successful patch and repair path in both arms while assisted token use and wall time were higher. It demonstrates discoverability and interaction, not performance benefit. diff --git a/experiments/situated-experience-benchmark-v1/CANDIDATE-SCREENING.md b/experiments/situated-experience-benchmark-v1/CANDIDATE-SCREENING.md new file mode 100644 index 0000000..fd43317 --- /dev/null +++ b/experiments/situated-experience-benchmark-v1/CANDIDATE-SCREENING.md @@ -0,0 +1,70 @@ +# Candidate screening contract + +Screening is completed and frozen before any arm output exists. Every inspected +candidate is recorded in `candidate-screening.json`, including rejections. A +candidate cannot be revived after outcomes are observed; it must enter a future +manifest revision with a new benchmark ID. + +All families require public provenance or an explicitly authorized private +corpus, a reproducible failing state, an objective evaluator, a bounded license +and privacy review, a source experience that predates the transfer task, and a +credible reason that situated knowledge could alter the repair path. Exclude +tasks whose answer is present in the prompt, whose human patch is reachable by +the agent, whose evaluator cannot distinguish a plausible false positive, or +whose environment cannot be staged before execution. + +## S1 dependency and version migration + +Accept only natural dependency, runtime, protocol, or metadata-representation +migrations with a version-locked pre-migration failure, a known human fix, and +visible plus controller-only regression coverage. Source and transfer must +share a migration invariant but differ in symptom and production patch. The +source fix must predate the transfer fix. Reject pure version-string bumps, +tasks requiring unstaged network downloads, tasks whose only oracle is static +syntax, and pairs whose compact experience would reveal the transfer patch. + +## S2 CI and deployment failures + +Require a reproducible failing job or deployment stage, a frozen runner image +and matrix, sanitized logs, and an evaluator that can rerun the relevant job in +the same substrate. Source and transfer must share an environmental or pipeline +failure invariant without sharing the same workflow edit. Reject provider-only +failures that cannot be replayed, secret-dependent tasks without an approved +broker, and green local substitutes for a failing hosted job. + +## S3 cross-module regressions + +Require a change in one module with an objectively failing consumer in at least +one other module. The hidden suite must exercise consequences beyond the +obvious edit site. Source experience must encode ownership or contract evidence, +not a filename. Reject single-file failures, tasks with no downstream oracle, +and tasks where the prompt names every affected module. + +## S4 Planner-Coder-Tester-Reviewer collaboration + +Require four independently logged roles, fixed handoff artifacts, bounded role +budgets, and evaluators for plan fidelity, patch correctness, test adequacy, and +review finding quality. Source experience may affect handoff content but cannot +contain role-specific answers. Reject tasks solvable without a handoff, roles +that share hidden state, and workflows where reviewer findings reach earlier +roles before their artifacts freeze. + +## S5 misleading repairs and repeated failure paths + +Require at least one historically plausible repair that passes a weak check but +fails a stronger registered oracle. Freeze detectors for repeated failed paths +before execution. Source experience must provide invalidating evidence and a +recovery principle, not the correct patch. Reject manufactured traps with no +public or recorded history and tasks whose false path is disclosed verbatim in +the task prompt. + +## S6 experience invalidation under environment drift + +Require paired environments where a once-valid experience becomes inapplicable +because of a frozen dependency, runtime, platform, or configuration change. The +correct treatment behavior must include rejection or abstention. Reject drift +that also changes the task oracle, environments that cannot be reconstructed, +and cases where the new environment merely repeats S1's original migration. + +S2-S6 are screening rules only in v1. They have no accepted tasks, fixtures, or +execution authorization. diff --git a/experiments/situated-experience-benchmark-v1/MEASUREMENT-CONTRACT.md b/experiments/situated-experience-benchmark-v1/MEASUREMENT-CONTRACT.md new file mode 100644 index 0000000..cc54999 --- /dev/null +++ b/experiments/situated-experience-benchmark-v1/MEASUREMENT-CONTRACT.md @@ -0,0 +1,78 @@ +# Common measurement and evaluation contract + +This contract applies unchanged to every future family. `arm-result.schema.json` +is the machine-readable record. Missing token telemetry is `null` with a reason; +no unavailable value is inferred. + +## Arm measurements + +- **Regression-free success:** the final production patch passes the visible + focused test and every controller-only regression test without changing any + test, task, experience, or evaluator file. +- **Attempts:** distinct non-empty production patch snapshots. Reformat-only + snapshots with the same normalized diff are one attempt. +- **Completed commands:** completed agent command-execution events. Controller + preflights and evaluator commands are recorded as tests but not agent commands. +- **Tests run:** every detected agent test invocation plus focused, hidden, and + broader evaluator commands, with scope and result. +- **Files inspected and changed:** repository-relative paths only. Inspection is + derived from command arguments and structured agent output; changed files are + authoritative from Git. +- **Patch size:** added lines, deleted lines, and changed-file count from the + production diff. +- **Wall time:** monotonic milliseconds from agent process start through exit; + evaluator time is separate and cannot qualify a result. +- **Tokens:** input and output usage emitted by the model runner, or `null` with + the exact unavailability reason. +- **Historical paths repeated:** pre-registered path IDs whose patterns match + the stated approach, first patch, or final patch. +- **Environment assumptions checked:** each registered assumption, whether it + was checked, and local evidence. +- **Experience disposition:** every experience is recorded as retrieved, used, + rejected, or abstained with a reason. Control records an abstention because no + experience is available. +- **Negative transfer:** paired evaluator result; true when treatment introduces + a regression, loses regression-free success, or materially worsens at least + two effort measures without a correctness gain. +- **Evaluator findings:** focused and hidden outcomes, test-file protection, + prohibited artifact changes, patch applicability, and protocol deviations. + +## Pair and benchmark evaluation + +Evaluate a replicate only after both modes have terminated. Evaluate each pair +across its three replicates before computing benchmark medians; replicates are +not independent tasks. A repair-path improvement is interpretable only when a +pre-registered historical path is avoided, an environment assumption is checked +earlier, an inapplicable experience is explicitly rejected, or correctness is +preserved while at least two of attempts, commands, or tests improve. Token and +wall-time deltas alone are not repair-path improvements. + +Treatment token overhead is `(median_treatment - median_control) / +median_control`, using input plus output tokens when both are available. If more +than half of arms lack token telemetry, the overhead criterion is unevaluable +and promotion fails closed. + +## Pre-registered promotion criteria + +All criteria are required: + +1. no reduction in regression-free success; +2. at least one interpretable repair-path improvement; +3. no leakage or contamination; +4. treatment token overhead no greater than 30% at the median; +5. wall-time improvement alone cannot qualify as positive evidence. + +## Pre-registered stop conditions + +Stop the benchmark and do not interpret arm outcomes if: + +1. hidden evaluator data is reachable by an agent; +2. one arm can read another arm's artifacts; +3. treatment contains or reveals the transfer patch; +4. a task, threshold, or metric changes after outcomes are observed; +5. both accepted pairs are trivial one-shot tasks with identical repair paths; +6. treatment causes additional regressions. + +Infrastructure failure, model unavailability, or missing token telemetry is not +silently converted into a repair failure. It is recorded as a protocol finding +and evaluated under the fail-closed rules above. diff --git a/experiments/situated-experience-benchmark-v1/README.md b/experiments/situated-experience-benchmark-v1/README.md new file mode 100644 index 0000000..a80f5b9 --- /dev/null +++ b/experiments/situated-experience-benchmark-v1/README.md @@ -0,0 +1,79 @@ +# Situated Experience Benchmark v1 + +Situated Experience Benchmark v1 tests whether AEG helps when repair depends on +knowledge of version state, execution environment, historical failures, +cross-module consequences, multi-agent handoffs, and experience applicability. + +It extends the narrow public repair evidence without rewriting it: the prior +five-pair experiment verified all 10 arms, reduced median completed commands by +1, and regressed median wall time by about 18.2 seconds. That result did not +establish broader AEG effectiveness. No Situated Experience Benchmark arm has +run yet. + +The six ordered families are S1 dependency and version migration, S2 CI and +deployment failures, S3 cross-module regressions, S4 Planner-Coder-Tester- +Reviewer collaboration, S5 misleading repairs and repeated failure paths, and +S6 experience invalidation under environment drift. This bounded revision +implements and freezes S1 only; S2-S6 remain design-only screening contracts. + +## S1 contents + +Exactly two natural public source-transfer pairs are accepted: + +1. Scrapy CookieJar adaptation across Python 3 decoding and request-protocol + changes. +2. FastAPI request handling across Pydantic 1.x field representations. + +Each offline fixture is a dependency-free public extract tied to upstream bug, +commit, date, license, and human-fix evidence. The transfer workspace contains a +visible failure. Controller-only directories contain the human patch and broader +tests and are never packaged for an agent. Each treatment receives only the +eight allowed compact-experience fields; no transfer patch or evaluator fact is +present. + +## Validation and preflight + +These commands never invoke an agent: + +```sh +python3 experiments/situated-experience-benchmark-v1/run_benchmark.py validate +python3 experiments/situated-experience-benchmark-v1/run_benchmark.py preflight +python3 experiments/situated-experience-benchmark-v1/test_benchmark.py +``` + +`preflight` proves that every buggy source and transfer fails for its registered +reason, every hidden human patch passes visible and hidden suites, and generated +control and AEG-assisted bundles exclude evaluator, other-arm, and other-pair +data. Adversarial tests verify that injected evaluator files, credentials, +cross-arm sentinels, altered experiences, and fixture drift fail closed. + +## Deterministic replay interface + +The arm selector is explicit and frozen: + +```sh +python3 experiments/situated-experience-benchmark-v1/run_benchmark.py package-arm \ + --pair s1-01-scrapy-cookiejar --replicate 1 --mode control --output /tmp/seb-arm +python3 experiments/situated-experience-benchmark-v1/run_benchmark.py package-arm \ + --pair s1-01-scrapy-cookiejar --replicate 1 --mode aeg-assisted --output /tmp/seb-arm-aeg +``` + +An arm bundle contains a one-commit transfer workspace, an immutable envelope, +the structured-result schema, and the standalone worker. It must execute on a +fresh disposable runner that has only that bundle. Running an agent from this +controller checkout is prohibited because the tracked evaluator data is +readable here. The worker requires `SEB_DISPOSABLE_RUNNER=1` and a dedicated +`SEB_RUNNER_ROOT` whose sole child is the bundle before execution. Pairwise +evaluation occurs only after both bundles return. + +`schedule-s1` packages the frozen 12-arm plan (two pairs, two modes, three +replicates) without executing it. The frozen order comes from the registered +seed. Actual execution requires a disposable-runner coordinator and an +authenticated model broker that does not expose credentials to agent commands. + +## Evidence boundary + +Promotion and stop conditions are frozen in `s1-manifest.json` and explained in +`MEASUREMENT-CONTRACT.md`. Wall time is measured but cannot qualify positive +evidence. Negative, neutral, abstention, protocol-deviation, and infrastructure +outcomes must be retained. S1 cannot establish value for S2-S6. diff --git a/experiments/situated-experience-benchmark-v1/REPRODUCIBILITY.md b/experiments/situated-experience-benchmark-v1/REPRODUCIBILITY.md new file mode 100644 index 0000000..8599f38 --- /dev/null +++ b/experiments/situated-experience-benchmark-v1/REPRODUCIBILITY.md @@ -0,0 +1,105 @@ +# S1 reproducibility record + +The tracked manifest is the sole source of task identity, task inputs, fixture +hashes, public commits, experience payloads, seed, budgets, arm order, metrics, +promotion criteria, and stop conditions. Local `.aeg` data, chat history, +screenshots, caches, upstream working trees, and untracked files are excluded. + +## Frozen inputs + +- Manifest SHA-256: `95ce8de8aca5580c8be95613b6058baecf2d473d9241831657e2a939577919c9`. +- Freeze time: `2026-08-14T18:46:10Z`, before any arm execution. +- Pairs: 2; modes per pair: 2; replicates per mode: 3; planned arms: 12. +- Randomization seed: `situated-experience-benchmark-v1-s1-2026-08-14`. +- Arm budget: 900 seconds, 40 completed commands, 3 distinct production attempts. +- Model: `gpt-5.6-sol`; any unavailable input/output token telemetry remains + `null` with a reason. +- Fixtures: dependency-free public extracts staged in this directory; execution + and evaluator tests need no package or source download. + +`freeze.json` protects the manifest, family registry, candidate screening, +measurement contract, runner, standalone worker, schema tree, and fixture tree. +The runner refuses validation or packaging if any protected digest changes. + +## Reproduction commands + +Run from the repository root: + +```sh +python3 autonomous-lab/scripts/lab.py validate +python3 experiments/situated-experience-benchmark-v1/run_benchmark.py validate +python3 experiments/situated-experience-benchmark-v1/run_benchmark.py preflight +python3 experiments/situated-experience-benchmark-v1/test_benchmark.py +git diff --check +``` + +The preflight uses fresh temporary Git repositories. It confirms every buggy +source and transfer failure before applying any human patch, applies each +controller-only patch to a new copy, adds hidden tests only inside the evaluator +copy, and verifies the complete suite. Packaging audits every planned arm and +does not invoke Codex. + +## Isolation handoff + +`schedule-s1` creates one bundle per coordinate in frozen order. A bundle has +only a one-commit transfer workspace, `arm.json`, the agent-result schema, and +the standalone worker. The worker requires a dedicated `SEB_RUNNER_ROOT` whose +only child is that bundle, rejects credential-shaped environment variables, +and refuses an envelope whose `--mode` differs from `control` or +`aeg-assisted`. Controller-only hidden tests and human patches are used only by +`evaluate-arm` after the agent process has terminated. + +No arm has been executed while preparing this record. + +## 2026-08-14 execution continuation + +The continuation began from clean commit +`55a9edafdda8ef4b82fe643de17bd9054929adca`. The manifest recomputed to +`95ce8de8aca5580c8be95613b6058baecf2d473d9241831657e2a939577919c9`. +Draft PR 28 hosted run 31834107919 passed schema, fixture, controller isolation, +leakage, evaluator-access, extension, compilation, and packaging checks for +that commit. The hosted workflow does not run the site or autonomous-lab +suites; local runs passed 8 site tests and 64 autonomous-lab tests plus the +lab's validation, status, next-action, and report checks. + +The frozen schedule command generated 12 bundles at +`2026-08-14T19:39:06.329005+00:00`. Its exact tracked plan is +`execution/s1-execution-plan.json`, SHA-256 +`6e6a3b75102d03d804cf0b8e1f51b3b1194fe5e1c39802b9d0cc64043bb9582a`. + +Execution then stopped before the non-benchmark canary and before every arm. +Tracked repository state does not identify a disposable-runner provider or an +authenticated model broker that withholds credentials from agent commands. +The available Codex process shares the controller host and cannot satisfy the +controller, evaluator, cache, conversation, workspace, or process-isolation +requirements. It was not used as a substitute. Consequently zero model calls, +zero input/output tokens, zero hidden evaluations, and zero arm outcomes were +recorded. See `execution/substrate-preflight.json` and +`execution/RESULTS.md`. + +The frozen `decision-ledger.jsonl` remains unchanged because its digest is a +protected input in `freeze.json`. Post-freeze execution decisions continue its +hash chain in `execution/decision-ledger.jsonl`. + +## 2026-08-14 arm-substrate deviation + +Because zero arms and zero outcomes existed, execution infrastructure moved to +the tracked AEG Arm Execution Substrate without changing a frozen benchmark +input. The workflow pins `ubuntu-24.04` and container manifest +`sha256:519591d6871b7bc437060736b9f7456b8731f1499a57e22e6c285135ae657bf7`. +Each future matrix coordinate has its own fresh VM. The host copies only a +validated envelope and frozen one-commit task into a 32 MiB tmpfs repair +workspace with no bind mount or network; the host model client exposes four +strict functions whose operations run only through the container worker. + +Hosted run 31840751530 used runner image `ubuntu24` version +`20260810.271.1` and produced container image +`sha256:423c7064cc5a754bec9c1a40756a27bd1814f0ed428b6de68250bfbd6fe9f005`. +All four registered failure signatures and all four human patches passed in +that image. The canary passed 28 isolation and enforcement attempts, removed +its plaintext raw file, and executed zero benchmark arms. It remained blocked +before a model request because the repository had no Actions secrets named +`OPENAI_API_KEY` or `AEG_RAW_OUTPUT_CERT_PEM`. Therefore live control and +treatment telemetry, cost accounting, and encrypted artifact retention remain +unverified. The exact sanitized record is tracked in +`execution/substrate-preflight.json`. diff --git a/experiments/situated-experience-benchmark-v1/STATUS.md b/experiments/situated-experience-benchmark-v1/STATUS.md new file mode 100644 index 0000000..69475f8 --- /dev/null +++ b/experiments/situated-experience-benchmark-v1/STATUS.md @@ -0,0 +1,51 @@ +# Situated Experience Benchmark v1 status + +Last updated: 2026-08-14 (America/Los_Angeles). + +- Phase: S1 execution stopped at the real-substrate gate; infrastructure-blocked. +- Accepted S1 pairs: 2 exactly. +- Rejected S1 candidates: 5, all with recorded reasons. +- Planned arms: 12 (2 pairs × 2 modes × 3 replicates). +- Generated plan: 12 frozen arms; plan SHA-256 `6e6a3b75102d03d804cf0b8e1f51b3b1194fe5e1c39802b9d0cc64043bb9582a`. +- Executed arms: 0; one non-benchmark hosted substrate canary ran; no benchmark + hidden evaluation ran. +- S2-S6: screening rules only; no fixtures, manifests, or arms implemented. +- Fixture preflight: passed again on pinned hosted image + `sha256:423c7064cc5a754bec9c1a40756a27bd1814f0ed428b6de68250bfbd6fe9f005`; + four buggy failures matched registered reasons and four human patches passed + the complete registered suites. +- Isolation/leakage/evaluator-access preflight: passed for all 12 planned one-arm + bundles; 20 substrate unit tests and 28 hosted adversarial attempts passed. +- Manifest mutation after outcomes: prohibited. +- Frozen manifest SHA-256: `95ce8de8aca5580c8be95613b6058baecf2d473d9241831657e2a939577919c9`. +- Hosted Repair Lab CI: passed for infrastructure commit + `be072873311621bfc7f56606db70b2f8a40d5bb5`; the separate hosted substrate + job intentionally failed its readiness gate at the missing host inputs. +- Real substrate preflight: partially passed and still blocked. The reusable + GitHub-hosted controller, zero-bind tmpfs repair container, strict four-tool + bridge, separate evaluator, sanitizer, and encrypted-output path are + implemented. Hosted run 31840751530 passed 28 adversarial fixture, + isolation, resource, patch-export, evaluator-order, and schema checks. It + failed closed because Actions has no `OPENAI_API_KEY` or + `AEG_RAW_OUTPUT_CERT_PEM`, so model access, encryption, and live token/cost + telemetry remain unproved. +- Classification: `infrastructure-blocked`; promotion is not supported and + fails closed because correctness, repair-path, leakage, and token-overhead + criteria are not evaluable. + +The current public evidence remains narrow: 10/10 prior repair-lab arms passed, +median completed commands improved by 1, and median wall time regressed by about +18.2 seconds. No broad effectiveness claim is supported. + +The plan-generation command has now run exactly once: + +```sh +python3 experiments/situated-experience-benchmark-v1/run_benchmark.py schedule-s1 \ + --output /tmp/situated-experience-benchmark-v1-s1-95ce8de8 +``` + +The exact plan is tracked at `execution/s1-execution-plan.json`. There is no +safe next arm command until both required Actions secrets are configured and +the same hosted adversarial canary passes as recorded in +`execution/substrate-preflight.json`. Do not execute from the controller +checkout and do not expand to full historical dependency stacks yet. diff --git a/experiments/situated-experience-benchmark-v1/arm_worker.py b/experiments/situated-experience-benchmark-v1/arm_worker.py new file mode 100644 index 0000000..35f9c53 --- /dev/null +++ b/experiments/situated-experience-benchmark-v1/arm_worker.py @@ -0,0 +1,314 @@ +#!/usr/bin/env python3 +"""Standalone one-arm worker for a disposable Situated Experience runner.""" + +import argparse +import hashlib +import json +import os +import re +import select +import shlex +import shutil +import subprocess +import sys +import time +from pathlib import Path + + +EXPECTED_TOP_LEVEL = {"agent-result.schema.json", "arm.json", "arm_worker.py", "workspace"} +SECRET_NAME = re.compile(r"(TOKEN|SECRET|PASSWORD|API_KEY|ACTIONS_|GITHUB_)", re.IGNORECASE) + + +class WorkerError(RuntimeError): + pass + + +def sha256_bytes(value): + return hashlib.sha256(value).hexdigest() + + +def load_json(path): + with Path(path).open(encoding="utf-8") as handle: + return json.load(handle) + + +def run(args, cwd=None, env=None, timeout=120): + return subprocess.run( + args, + cwd=cwd, + env=env, + text=True, + capture_output=True, + check=False, + timeout=timeout, + ) + + +def git(workspace, *args): + result = run(["git", *args], cwd=workspace) + if result.returncode: + raise WorkerError(result.stderr.strip() or "git command failed") + return result.stdout + + +def assert_bundle(bundle, expected_mode=None): + actual = {path.name for path in bundle.iterdir()} + if actual != EXPECTED_TOP_LEVEL: + raise WorkerError(f"unexpected bundle entries: {sorted(actual ^ EXPECTED_TOP_LEVEL)}") + envelope = load_json(bundle / "arm.json") + if expected_mode and envelope.get("mode") != expected_mode: + raise WorkerError("--mode differs from immutable arm envelope") + if envelope.get("mode") not in ("control", "aeg-assisted"): + raise WorkerError("invalid arm mode") + workspace = bundle / "workspace" + if git(workspace, "remote").strip(): + raise WorkerError("arm workspace has a Git remote") + if git(workspace, "rev-list", "--all", "--count").strip() != "1": + raise WorkerError("arm workspace must have exactly one commit") + if git(workspace, "status", "--short").strip(): + raise WorkerError("arm workspace is not clean") + if envelope["mode"] == "control" and "experience" in envelope: + raise WorkerError("control envelope contains an experience") + if envelope["mode"] == "aeg-assisted" and set(envelope.get("experience", {})) != set(envelope["allowed_experience_fields"]): + raise WorkerError("assisted experience fields differ from the frozen allowlist") + return envelope + + +def probe(bundle): + envelope = assert_bundle(bundle) + exposed = sorted(name for name in os.environ if SECRET_NAME.search(name)) + allowed = {"GITHUB_ACTIONS"} if os.environ.get("GITHUB_ACTIONS") == "false" else set() + exposed = [name for name in exposed if name not in allowed] + if exposed: + raise WorkerError(f"credential-shaped environment names are exposed: {exposed}") + for forbidden in ("human.patch", "test_hidden.py", "evaluator.json", "prior-arm.patch", "prior-arm.log"): + if any(path.name == forbidden for path in bundle.rglob("*")): + raise WorkerError(f"evaluator or cross-arm artifact is reachable: {forbidden}") + runner_root_text = os.environ.get("SEB_RUNNER_ROOT") + if runner_root_text: + runner_root = Path(runner_root_text).resolve() + if bundle.parent != runner_root: + raise WorkerError("bundle is not the sole child of SEB_RUNNER_ROOT") + siblings = [path.name for path in runner_root.iterdir() if path.resolve() != bundle] + if siblings: + raise WorkerError(f"other-arm or controller artifacts are reachable: {sorted(siblings)}") + return {"arm_id": envelope["arm_id"], "status": "passed"} + + +def render_prompt(envelope): + prompt = envelope["task_prompt"] + prompt += ( + "\n\nBefore the first edit, state the intended production location and approach. " + "Check the relevant runtime or dependency representation locally. " + "Do not modify tests. Work only in this repository. Return the required " + "structured result, including environment assumptions checked." + ) + if envelope["mode"] == "aeg-assisted": + prompt += "\n\nAEG retrieved this compact experience. Use it only if local evidence satisfies its applicability conditions:\n" + for key in envelope["allowed_experience_fields"]: + prompt += f"\n{key}: {envelope['experience'][key]}" + else: + prompt += "\n\nNo AEG experience is available in the control mode. Record experience_disposition as abstained." + return prompt + + +def parse_event_metrics(events, workspace, public_command): + commands = [] + attempts = [] + inspected = set() + tests = [] + usage = {} + known = [path.relative_to(workspace).as_posix() for path in workspace.rglob("*") if path.is_file() and ".git" not in path.parts] + for event in events: + if event.get("type") == "turn.completed": + usage = event.get("usage") or usage + if event.get("type") != "item.completed": + continue + item = event.get("item") or {} + if item.get("type") == "command_execution" and item.get("command"): + value = item["command"] + commands.append(value) + for relative in known: + if relative in value: + inspected.add(relative) + if re.search(r"(?:pytest|unittest|test_[A-Za-z0-9_./-]*\.py|python\d*\s+[^\n]*test)", value): + tests.append({"command": value, "scope": "agent", "passed": item.get("exit_code", 0) == 0}) + if item.get("type") == "file_change": + changed = sorted(change.get("path", "") for change in item.get("changes", []) if change.get("path")) + attempts.append(changed) + input_tokens = usage.get("input_tokens") + output_tokens = usage.get("output_tokens") + return { + "commands": commands, + "attempts": attempts, + "files_inspected": sorted(inspected), + "tests": tests, + "tokens": { + "input": input_tokens if isinstance(input_tokens, int) else None, + "output": output_tokens if isinstance(output_tokens, int) else None, + "unavailable_reason": None if isinstance(input_tokens, int) and isinstance(output_tokens, int) else "runner event stream did not expose complete token usage", + }, + } + + +def patch_stats(diff): + added = deleted = 0 + files = [] + for line in diff.splitlines(): + if line.startswith("diff --git a/"): + files.append(line.split(" b/", 1)[1]) + elif line.startswith("+") and not line.startswith("+++"): + added += 1 + elif line.startswith("-") and not line.startswith("---"): + deleted += 1 + return {"added_lines": added, "deleted_lines": deleted, "files": len(set(files))}, sorted(set(files)) + + +def read_events(path): + events = [] + for line in path.read_text(encoding="utf-8", errors="replace").splitlines(): + try: + events.append(json.loads(line)) + except json.JSONDecodeError: + continue + return events + + +def execute(bundle, output, mode, codex): + if os.environ.get("SEB_DISPOSABLE_RUNNER") != "1": + raise WorkerError("execution requires SEB_DISPOSABLE_RUNNER=1 on a one-arm host") + if not os.environ.get("SEB_RUNNER_ROOT"): + raise WorkerError("execution requires a dedicated SEB_RUNNER_ROOT containing only this bundle") + envelope = assert_bundle(bundle, mode) + probe(bundle) + output.mkdir(parents=True, exist_ok=False) + workspace = bundle / "workspace" + events_path = output / "events.jsonl" + stderr_path = output / "stderr.log" + structured_path = output / "agent-result.json" + command = [ + str(codex), "exec", "--ephemeral", "--ignore-user-config", + "--model", envelope["model"], "--sandbox", "workspace-write", "--json", + "--output-schema", str(bundle / "agent-result.schema.json"), + "-o", str(structured_path), render_prompt(envelope), + ] + started = time.monotonic() + completed_commands = 0 + attempt_hashes = [] + timed_out = budget_exceeded = False + with events_path.open("w", encoding="utf-8") as stream, stderr_path.open("w", encoding="utf-8") as error_stream: + process = subprocess.Popen(command, cwd=workspace, stdout=subprocess.PIPE, stderr=error_stream, text=True, bufsize=1) + assert process.stdout is not None + while process.poll() is None: + if time.monotonic() - started > envelope["budget"]["wall_time_seconds"]: + timed_out = True + process.terminate() + break + ready, _, _ = select.select([process.stdout], [], [], 0.25) + if not ready: + continue + line = process.stdout.readline() + if not line: + continue + stream.write(line) + stream.flush() + try: + event = json.loads(line) + except json.JSONDecodeError: + continue + item = event.get("item") or {} + if event.get("type") == "item.completed" and item.get("type") == "command_execution": + completed_commands += 1 + if event.get("type") == "item.completed" and item.get("type") == "file_change": + snapshot = git(workspace, "diff", "--binary", "--", ".") + snapshot_hash = sha256_bytes(snapshot.encode()) if snapshot else None + if snapshot_hash and snapshot_hash not in attempt_hashes: + attempt_hashes.append(snapshot_hash) + if completed_commands > envelope["budget"]["max_completed_commands"] or len(attempt_hashes) > envelope["budget"]["max_attempts"]: + budget_exceeded = True + process.terminate() + break + try: + process.wait(timeout=10) + except subprocess.TimeoutExpired: + process.kill() + process.wait() + for line in process.stdout: + stream.write(line) + wall_time_ms = round((time.monotonic() - started) * 1000) + diff = git(workspace, "diff", "--binary", "--", ".") + final_hash = sha256_bytes(diff.encode()) if diff else None + if final_hash and final_hash not in attempt_hashes: + attempt_hashes.append(final_hash) + (output / "patch.diff").write_text(diff, encoding="utf-8") + metrics = parse_event_metrics(read_events(events_path), workspace, envelope["public_test_command"]) + public = run(shlex.split(envelope["public_test_command"]), cwd=workspace, timeout=120) + metrics["tests"].append({"command": envelope["public_test_command"], "scope": "focused", "passed": public.returncode == 0}) + stats, changed_files = patch_stats(diff) + structured = load_json(structured_path) if structured_path.is_file() else {} + if mode == "control": + experiences = [{"experience_id": None, "disposition": "abstained", "reason": "control mode has no AEG experience"}] + else: + disposition = structured.get("experience_disposition", "abstained") + experiences = [ + {"experience_id": envelope["experience_id"], "disposition": "retrieved", "reason": "frozen treatment payload delivered"}, + {"experience_id": envelope["experience_id"], "disposition": disposition, "reason": structured.get("experience_reason", "structured disposition unavailable")}, + ] + result = { + "schema_version": "1.0.0", + "benchmark_id": envelope["benchmark_id"], + "family": "S1", + "pair_id": envelope["pair_id"], + "replicate": envelope["replicate"], + "mode": mode, + "evaluation_status": "captured", + "input_hashes": envelope["input_hashes"], + "budget": envelope["budget"], + "regression_free_success": None, + "attempts": len(attempt_hashes), + "completed_commands": len(metrics["commands"]), + "tests_run": metrics["tests"], + "files_inspected": metrics["files_inspected"], + "files_changed": changed_files, + "patch_size": stats, + "wall_time_ms": wall_time_ms, + "tokens": metrics["tokens"], + "failed_historical_paths_repeated": [], + "environment_assumptions_checked": structured.get("environment_assumptions_checked", []), + "experiences": experiences, + "negative_transfer": None, + "evaluator_findings": ["hidden evaluation pending"], + "limitations": [item for item, present in (("agent timed out", timed_out), ("agent exceeded a frozen command or attempt budget", budget_exceeded), ("Codex process exited non-zero", process.returncode != 0)) if present], + } + (output / "arm-result.json").write_text(json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8") + return result + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest="action", required=True) + check = sub.add_parser("probe") + check.add_argument("--bundle", default=".") + run_parser = sub.add_parser("execute") + run_parser.add_argument("--bundle", default=".") + run_parser.add_argument("--output", required=True) + run_parser.add_argument("--mode", required=True, choices=("control", "aeg-assisted")) + run_parser.add_argument("--codex", default=shutil.which("codex")) + args = parser.parse_args() + bundle = Path(args.bundle).resolve() + if args.action == "probe": + print(json.dumps(probe(bundle), indent=2, sort_keys=True)) + return 0 + if not args.codex: + raise WorkerError("Codex executable not found") + result = execute(bundle, Path(args.output).resolve(), args.mode, Path(args.codex).resolve()) + print(json.dumps({"arm_id": load_json(bundle / "arm.json")["arm_id"], "captured": result["evaluation_status"]}, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except (WorkerError, subprocess.TimeoutExpired) as error: + print(f"situated arm worker error: {error}", file=sys.stderr) + raise SystemExit(2) diff --git a/experiments/situated-experience-benchmark-v1/candidate-screening.json b/experiments/situated-experience-benchmark-v1/candidate-screening.json new file mode 100644 index 0000000..d0b6a61 --- /dev/null +++ b/experiments/situated-experience-benchmark-v1/candidate-screening.json @@ -0,0 +1,116 @@ +{ + "schema_version": "1.0.0", + "benchmark_id": "situated-experience-benchmark-v1", + "screened_before_execution": true, + "candidates": [ + { + "candidate_id": "cand-scrapy-cookiejar-python3", + "family": "S1", + "status": "accepted", + "title": "Scrapy CookieJar adapter across Python 3 protocol changes", + "public_evidence": [ + "BugsInPy scrapy-31 and scrapy-19", + "https://github.com/scrapy/scrapy/commit/dba7e39f61cbe2c22d3c9064f32f6e36d74f14b2", + "https://github.com/scrapy/scrapy/commit/1f743996ff00a7b728d59b93d0967e1eb50072f0" + ], + "decision_reasons": [ + "The source fix predates the transfer fix and both are natural public repairs.", + "The pair shares one Python 3 CookieJar adapter boundary but source decoding and transfer protocol-member patches are non-identical.", + "A dependency-free public extract reproduces the registered source and transfer failures offline, and hidden evaluator coverage is available." + ], + "pair_id": "s1-01-scrapy-cookiejar" + }, + { + "candidate_id": "cand-fastapi-pydantic-representations", + "family": "S1", + "status": "accepted", + "title": "FastAPI request handling across Pydantic field representations", + "public_evidence": [ + "BugsInPy fastapi-11 and fastapi-6", + "https://github.com/fastapi/fastapi/commit/06eb4219345a77d23484528c9d164eb8d2097fec", + "https://github.com/fastapi/fastapi/commit/874d24181e779ebc6e1c52afb7d6598f863fd6a8" + ], + "decision_reasons": [ + "The source fix predates the transfer fix and both were independently repaired upstream.", + "Both require reasoning about Pydantic 1.x metadata representations, while source Union classification and transfer concrete-form extraction use non-identical code paths and patches.", + "Visible and hidden dependency-free tests preserve the upstream failing representation and execute offline." + ], + "pair_id": "s1-02-fastapi-pydantic" + }, + { + "candidate_id": "cand-pine-checkout-v4-v6", + "family": "S1", + "status": "rejected", + "title": "Pine actions/checkout v4 to v6 workflow replacement", + "public_evidence": [ + "dogfood/self-consumption-batch-01/candidates/02-api-dependency-migration.json", + "batonogov/pine issue 122" + ], + "decision_reasons": [ + "The tracked evidence records only static reference and YAML checks, not GitHub-hosted Node 24 behavior.", + "The migration is an identical three-line version replacement and would add a trivial path with no blind behavioral evaluator." + ], + "rejection_code": "trivial_static_migration" + }, + { + "candidate_id": "cand-sanic-asyncio-server", + "family": "S1", + "status": "rejected", + "title": "Sanic AsyncioServer parity with Python 3.7", + "public_evidence": [ + "experiences/work-queue/runs/AM-01/STATUS.md", + "BugsInPy sanic-2" + ], + "decision_reasons": [ + "The tracked record is partial: complete historical suite coverage and older asyncio or uvloop compatibility remain unverified.", + "The recorded candidate omits the human repair's compatibility behavior, so the current fixture cannot support regression-free blind evaluation." + ], + "rejection_code": "regression_coverage_incomplete" + }, + { + "candidate_id": "cand-keras-tfoptimizer", + "family": "S1", + "status": "rejected", + "title": "Keras TFOptimizer named-argument contract", + "public_evidence": [ + "experiences/work-queue/README.md#am-02--keras-tfoptimizer-named-argument-contract", + "BugsInPy keras-4" + ], + "decision_reasons": [ + "The historical TensorFlow and Keras dependency stack was not staged and is not reproducible in the bounded offline environment.", + "No accepted, earlier non-identical source migration was established before freeze." + ], + "rejection_code": "oracle_not_reproducible_offline" + }, + { + "candidate_id": "cand-ansible-numeric-version", + "family": "S1", + "status": "rejected", + "title": "Ansible resolver with numeric dependency versions", + "public_evidence": [ + "experiences/work-queue/README.md#am-03--ansible-dependency-resolver-accepts-non-string-versions", + "BugsInPy ansible-6" + ], + "decision_reasons": [ + "The task is a configuration-type normalization repair, but no source-before-transfer pair with a shared situated migration principle was verified.", + "Accepting a transfer-only task would violate the source-transfer design." + ], + "rejection_code": "source_transfer_relation_missing" + }, + { + "candidate_id": "cand-scrapy-location-normalization", + "family": "S1", + "status": "rejected", + "title": "Scrapy redirect Location normalization", + "public_evidence": [ + "experiments/natural-transfer-benchmark/manifest.json task nt-02", + "BugsInPy scrapy-10 and scrapy-3" + ], + "decision_reasons": [ + "The task is a URL-semantics regression and does not depend on dependency or version migration state.", + "It remains eligible for a future cross-module or misleading-path family, but not S1." + ], + "rejection_code": "family_mismatch" + } + ] +} diff --git a/experiments/situated-experience-benchmark-v1/decision-ledger.jsonl b/experiments/situated-experience-benchmark-v1/decision-ledger.jsonl new file mode 100644 index 0000000..051b3f1 --- /dev/null +++ b/experiments/situated-experience-benchmark-v1/decision-ledger.jsonl @@ -0,0 +1,4 @@ +{"decision":"Treat the prior five-pair repair experiment as narrow evidence only.","event_sha256":"edc6618f96683f22716c0b272ba006947cf99953bee4390da0b75869240d818a","event_type":"evidence_boundary_preserved","evidence":["experiments/public-repair-lab/RESULTS.md","experiments/public-repair-lab/results/v0.1.3-paired-results.json"],"previous_event_sha256":null,"rationale":"All 10 arms passed, median completed commands improved by 1, and median wall time regressed by about 18.2 seconds; these observations do not establish broad AEG effectiveness.","sequence":1,"timestamp":"2026-08-14T18:20:00Z"} +{"decision":"Order six Situated Experience families and implement only S1 in this bounded revision.","event_sha256":"591f21ce93a137c3fc39463489eace8169665174b3db086b50ec994c6970c0a7","event_type":"staged_design_registered","evidence":["experiments/situated-experience-benchmark-v1/registry.json","experiments/situated-experience-benchmark-v1/CANDIDATE-SCREENING.md"],"previous_event_sha256":"edc6618f96683f22716c0b272ba006947cf99953bee4390da0b75869240d818a","rationale":"The staged design separates version, CI, cross-module, collaboration, misleading-path, and drift claims so S1 cannot stand in for unexecuted families.","sequence":2,"timestamp":"2026-08-14T18:30:00Z"} +{"decision":"Accept exactly the Scrapy CookieJar and FastAPI Pydantic pairs; reject all five other screened candidates.","event_sha256":"c6d4264113f238afd92f0977253d255e5a5fe37ffe0885cc9eccf32ee34789fa","event_type":"s1_candidates_screened","evidence":["experiments/situated-experience-benchmark-v1/candidate-screening.json"],"previous_event_sha256":"591f21ce93a137c3fc39463489eace8169665174b3db086b50ec994c6970c0a7","rationale":"Both accepted pairs have source-before-transfer public fixes, non-identical patches, offline failing fixtures, hidden regression coverage, and compact experiences that do not reveal transfer patches.","sequence":3,"timestamp":"2026-08-14T18:40:00Z"} +{"decision":"Freeze S1 inputs, order, budgets, metrics, promotion criteria, and stop conditions before executing any arm.","event_sha256":"29a459be81dc2c88729941859c65a43f8d879a7ff840f8e392d730c3e5cc4844","event_type":"manifest_frozen","evidence":["experiments/situated-experience-benchmark-v1/s1-manifest.json","experiments/situated-experience-benchmark-v1/MEASUREMENT-CONTRACT.md"],"previous_event_sha256":"c6d4264113f238afd92f0977253d255e5a5fe37ffe0885cc9eccf32ee34789fa","rationale":"Post-outcome task, metric, threshold, fixture, experience, or controller changes would contaminate causal interpretation.","sequence":4,"timestamp":"2026-08-14T18:46:10Z"} diff --git a/experiments/situated-experience-benchmark-v1/execution/PROTOCOL-DEVIATION-ARM-SUBSTRATE.md b/experiments/situated-experience-benchmark-v1/execution/PROTOCOL-DEVIATION-ARM-SUBSTRATE.md new file mode 100644 index 0000000..0eeadb1 --- /dev/null +++ b/experiments/situated-experience-benchmark-v1/execution/PROTOCOL-DEVIATION-ARM-SUBSTRATE.md @@ -0,0 +1,33 @@ +# Infrastructure-only protocol deviation: container arm substrate + +Recorded: 2026-08-14, before any Situated Experience Benchmark arm. + +The original controller-host execution substrate was rejected before 0/12 S1 +arms. No task outcome, transfer patch, evaluator result, treatment comparison, +or benchmark metric had been observed. This deviation therefore changes only +the execution substrate. + +The frozen manifest, task pairs, prompts, compact experiences, modes, seeds, +budgets, arm ordering, measurements, promotion criteria, stop conditions, and +execution-plan hash remain unchanged. The authoritative hashes remain: + +- manifest: `95ce8de8aca5580c8be95613b6058baecf2d473d9241831657e2a939577919c9`; +- execution plan: `6e6a3b75102d03d804cf0b8e1f51b3b1194fe5e1c39802b9d0cc64043bb9582a`. + +Execution moves to a host-controller / pinned Linux repair-container boundary. +The GitHub-hosted Ubuntu VM may access the job-scoped model credential. The +model has no host tools: its four strict tool calls are validated by the host +and executed only in a networkless container. The controller copies one +sanitized envelope and one task into a hard-size-limited workspace tmpfs; the +container has no host bind mount. Hidden evaluation starts in a separate +container only after repair termination. Raw transcripts and patches are +encrypted to an externally held public-key recipient before artifact upload; +only schema-validated metrics are public evidence. + +The pinned base is +`python:3.12.11-slim-bookworm@sha256:519591d6871b7bc437060736b9f7456b8731f1499a57e22e6c285135ae657bf7`. +All four buggy source/transfer seeds, expected failure signatures, hidden suites, +and human patches must pass again on that image. S1 remains +`infrastructure-blocked` until the complete adversarial canary passes on the +same GitHub-hosted runner and container configuration. Mocked or local tests do +not establish readiness. diff --git a/experiments/situated-experience-benchmark-v1/execution/RESULTS.md b/experiments/situated-experience-benchmark-v1/execution/RESULTS.md new file mode 100644 index 0000000..52cad6f --- /dev/null +++ b/experiments/situated-experience-benchmark-v1/execution/RESULTS.md @@ -0,0 +1,47 @@ +# S1 execution result + +Situated Experience Benchmark v1 S1 is **infrastructure-blocked**. This is an +S1 mechanism pilot using faithful public extracts, not full historical +dependency stacks. + +The exact frozen 12-arm plan was generated after hosted CI passed. A reusable +host-controller and disposable-container substrate was subsequently built and +tested on GitHub-hosted Ubuntu. Its non-benchmark canary passed 28 fixture, +isolation, resource, patch-export, evaluator-order, and sanitizer attempts. +The gate remained blocked because the repository has neither required Actions +secret: `OPENAI_API_KEY` for the host controller and +`AEG_RAW_OUTPUT_CERT_PEM` for the external encryption recipient. No model call +or benchmark arm was attempted. + +## Arm accounting + +- Planned: 12. +- Started: 0. +- Completed: 0. +- Task failures: 0. +- Infrastructure-failed arm records: 0; the substrate failed before an arm was + started, so no task outcome was manufactured. +- Hidden evaluations: 0. +- Non-benchmark model calls and input/output tokens: 0 because the hosted + credential was absent. +- Recorded model cost: $0. + +There are no per-pair or aggregate repair results. Regression-free success, +attempts, completed commands, tests, files inspected, patch size, repeated +historical paths, negative transfer, experience disposition, environment +assumptions, token overhead, and median wall time are all not evaluable. + +## Mechanical decision + +Promotion is not supported and fails closed. No correctness comparison or +interpretable repair-path improvement exists, and treatment token overhead was +not measured. The real hosted container boundary passed its adversarial checks, +but live credential brokering, encrypted retention, and token telemetry were +not demonstrated. No frozen benchmark stop condition was triggered by an +observed arm outcome because no arm ran; instead, the prerequisite substrate +gate blocked execution. + +Do not expand S1 to full historical dependency stacks yet. First configure the +two host-only workflow inputs and rerun the same canary to demonstrate live +model access, encrypted retention, and token telemetry for both modes. S2-S6 +remain unstarted. diff --git a/experiments/situated-experience-benchmark-v1/execution/cost-ledger.jsonl b/experiments/situated-experience-benchmark-v1/execution/cost-ledger.jsonl new file mode 100644 index 0000000..26dbf57 --- /dev/null +++ b/experiments/situated-experience-benchmark-v1/execution/cost-ledger.jsonl @@ -0,0 +1,2 @@ +{"arms_completed":0,"arms_started":0,"benchmark_id":"situated-experience-benchmark-v1","cost_usd":0,"event_type":"execution_blocked_before_model_use","family":"S1","input_tokens":0,"model":"gpt-5.6-sol","model_calls":0,"output_tokens":0,"raw_model_outputs_retained":0,"reason":"No compliant disposable runner and credential broker were available; no canary or benchmark model call was made.","timestamp":"2026-08-14T19:42:01Z"} +{"arms_completed":0,"arms_started":0,"benchmark_id":"situated-experience-benchmark-v1","cost_usd":0,"event_type":"hosted_canary_blocked_before_model_use","family":"S1","input_tokens":0,"model":"gpt-5.6-sol","model_calls":0,"output_tokens":0,"raw_model_outputs_retained":0,"reason":"Hosted isolation and fixture checks ran, but no OPENAI_API_KEY or AEG_RAW_OUTPUT_CERT_PEM Actions secret was configured; no model request was attempted and plaintext was removed.","timestamp":"2026-08-14T21:04:18Z"} diff --git a/experiments/situated-experience-benchmark-v1/execution/decision-ledger.jsonl b/experiments/situated-experience-benchmark-v1/execution/decision-ledger.jsonl new file mode 100644 index 0000000..c63eb7f --- /dev/null +++ b/experiments/situated-experience-benchmark-v1/execution/decision-ledger.jsonl @@ -0,0 +1,2 @@ +{"decision":"Stop S1 before the non-benchmark canary and every benchmark arm; classify this continuation as infrastructure-blocked.","event_sha256":"bffa2798784a6c73f5d8d2f36532adb84d47ba2bbc03fe85951fa1b29d6e4308","event_type":"execution_substrate_blocked","evidence":["experiments/situated-experience-benchmark-v1/execution/substrate-preflight.json","experiments/situated-experience-benchmark-v1/execution/s1-execution-plan.json","experiments/situated-experience-benchmark-v1/execution/evidence-index.json"],"previous_event_sha256":"29a459be81dc2c88729941859c65a43f8d879a7ff840f8e392d730c3e5cc4844","rationale":"Tracked state provides a deterministic one-arm worker but no actual disposable-runner coordinator, authenticated credential broker, private raw-output sink, or sanitized result return channel. The available controller-host process fails the controller and shared-state isolation requirements, so using it would violate the preregistration.","sequence":5,"timestamp":"2026-08-14T19:42:01Z"} +{"decision":"Keep S1 infrastructure-blocked: the hosted container boundary passed, but live model credential, encryption, and telemetry proof remain unavailable.","event_sha256":"310c752322e918c1cd5effe24a63401401018c5a1dc90b69e1e41ab928d3b8c0","event_type":"hosted_substrate_blocked_at_credential_gate","evidence":["experiments/situated-experience-benchmark-v1/execution/substrate-preflight.json","experiments/situated-experience-benchmark-v1/execution/PROTOCOL-DEVIATION-ARM-SUBSTRATE.md","https://github.com/yao23/agent-experience-graph/actions/runs/31840751530"],"previous_event_sha256":"bffa2798784a6c73f5d8d2f36532adb84d47ba2bbc03fe85951fa1b29d6e4308","rationale":"On the pinned GitHub-hosted Ubuntu and container image, all four fixture failures and all four human patches passed, and 28 adversarial isolation, resource, export, evaluator, and sanitizer attempts passed. GitHub Actions has neither OPENAI_API_KEY nor AEG_RAW_OUTPUT_CERT_PEM configured, so the controller could not call the model, encrypt raw output, or demonstrate control/treatment token telemetry. Zero benchmark arms ran.","sequence":6,"timestamp":"2026-08-14T21:04:18Z"} diff --git a/experiments/situated-experience-benchmark-v1/execution/evidence-index.json b/experiments/situated-experience-benchmark-v1/execution/evidence-index.json new file mode 100644 index 0000000..82295a8 --- /dev/null +++ b/experiments/situated-experience-benchmark-v1/execution/evidence-index.json @@ -0,0 +1,84 @@ +{ + "schema_version": "1.0.0", + "benchmark_id": "situated-experience-benchmark-v1", + "family": "S1", + "implementation_commit": "be072873311621bfc7f56606db70b2f8a40d5bb5", + "manifest_sha256": "95ce8de8aca5580c8be95613b6058baecf2d473d9241831657e2a939577919c9", + "execution_plan": { + "path": "execution/s1-execution-plan.json", + "sha256": "6e6a3b75102d03d804cf0b8e1f51b3b1194fe5e1c39802b9d0cc64043bb9582a", + "arms": 12 + }, + "hosted_ci": { + "status": "success", + "run_id": 31840751520, + "job_id": 94896861600, + "run_url": "https://github.com/yao23/agent-experience-graph/actions/runs/31840751520", + "job_url": "https://github.com/yao23/agent-experience-graph/actions/runs/31840751520/job/94896861600", + "covered": [ + "public and benchmark schemas", + "S1 fixture and human-patch preflight", + "S1 packaging, isolation, leakage, and evaluator-access controller tests", + "extension tests and compilation", + "extension packaging" + ], + "not_covered": [ + "site regression suite", + "autonomous-lab validation and regression suite" + ] + }, + "hosted_substrate": { + "status": "blocked_at_missing_host_inputs", + "run_id": 31840751530, + "job_id": 94896861841, + "run_url": "https://github.com/yao23/agent-experience-graph/actions/runs/31840751530", + "job_url": "https://github.com/yao23/agent-experience-graph/actions/runs/31840751530/job/94896861841", + "sanitized_artifact_id": 9234146186, + "sanitized_artifact_sha256": "3a515538ae715d82a3fee624a4a838b1a3ca0bdbe2eaa6fc65ab1bf90c2465ae", + "passed": [ + "pinned image build", + "four buggy failure signatures", + "four human patches and registered suites", + "28 adversarial isolation and enforcement attempts" + ], + "blocked": [ + "host model call and both-mode token telemetry", + "encrypted raw artifact retention" + ], + "reason": "OPENAI_API_KEY and AEG_RAW_OUTPUT_CERT_PEM are not configured as Actions secrets." + }, + "local_validation": [ + { + "command": "python3 scripts/test_site.py", + "result": "passed", + "tests": 8 + }, + { + "command": "python3 autonomous-lab/scripts/lab.py validate --base-ref origin/main", + "result": "passed", + "ledger_events": 17 + }, + { + "command": "python3 autonomous-lab/scripts/lab.py status && python3 autonomous-lab/scripts/lab.py next && python3 autonomous-lab/scripts/lab.py report --check", + "result": "passed" + }, + { + "command": "python3 -m unittest discover -s autonomous-lab/scripts/tests -p 'test_*.py' -v", + "result": "passed", + "tests": 64 + }, + { + "command": "python3 -m unittest discover -s infrastructure/aeg-arm-execution-substrate/tests -p 'test_*.py' -v", + "result": "passed", + "tests": 20 + }, + { + "command": "git diff --check", + "result": "passed" + } + ], + "substrate_preflight": "execution/substrate-preflight.json", + "cost_ledger": "execution/cost-ledger.jsonl", + "execution_decision_ledger": "execution/decision-ledger.jsonl", + "result_summary": "execution/RESULTS.md" +} diff --git a/experiments/situated-experience-benchmark-v1/execution/s1-execution-plan.json b/experiments/situated-experience-benchmark-v1/execution/s1-execution-plan.json new file mode 100644 index 0000000..d65b420 --- /dev/null +++ b/experiments/situated-experience-benchmark-v1/execution/s1-execution-plan.json @@ -0,0 +1,70 @@ +{ + "arm_count": 12, + "arms": [ + { + "arm_id": "s1-01-scrapy-cookiejar--r01--aeg-assisted", + "mode": "aeg-assisted", + "relative_bundle": "arms/s1-01-scrapy-cookiejar--r01--aeg-assisted" + }, + { + "arm_id": "s1-01-scrapy-cookiejar--r01--control", + "mode": "control", + "relative_bundle": "arms/s1-01-scrapy-cookiejar--r01--control" + }, + { + "arm_id": "s1-01-scrapy-cookiejar--r02--aeg-assisted", + "mode": "aeg-assisted", + "relative_bundle": "arms/s1-01-scrapy-cookiejar--r02--aeg-assisted" + }, + { + "arm_id": "s1-01-scrapy-cookiejar--r02--control", + "mode": "control", + "relative_bundle": "arms/s1-01-scrapy-cookiejar--r02--control" + }, + { + "arm_id": "s1-01-scrapy-cookiejar--r03--aeg-assisted", + "mode": "aeg-assisted", + "relative_bundle": "arms/s1-01-scrapy-cookiejar--r03--aeg-assisted" + }, + { + "arm_id": "s1-01-scrapy-cookiejar--r03--control", + "mode": "control", + "relative_bundle": "arms/s1-01-scrapy-cookiejar--r03--control" + }, + { + "arm_id": "s1-02-fastapi-pydantic--r01--control", + "mode": "control", + "relative_bundle": "arms/s1-02-fastapi-pydantic--r01--control" + }, + { + "arm_id": "s1-02-fastapi-pydantic--r01--aeg-assisted", + "mode": "aeg-assisted", + "relative_bundle": "arms/s1-02-fastapi-pydantic--r01--aeg-assisted" + }, + { + "arm_id": "s1-02-fastapi-pydantic--r02--control", + "mode": "control", + "relative_bundle": "arms/s1-02-fastapi-pydantic--r02--control" + }, + { + "arm_id": "s1-02-fastapi-pydantic--r02--aeg-assisted", + "mode": "aeg-assisted", + "relative_bundle": "arms/s1-02-fastapi-pydantic--r02--aeg-assisted" + }, + { + "arm_id": "s1-02-fastapi-pydantic--r03--control", + "mode": "control", + "relative_bundle": "arms/s1-02-fastapi-pydantic--r03--control" + }, + { + "arm_id": "s1-02-fastapi-pydantic--r03--aeg-assisted", + "mode": "aeg-assisted", + "relative_bundle": "arms/s1-02-fastapi-pydantic--r03--aeg-assisted" + } + ], + "benchmark_id": "situated-experience-benchmark-v1", + "created_at": "2026-08-14T19:39:06.329005+00:00", + "execution_requirement": "Copy exactly one relative_bundle to each fresh disposable runner, set SEB_DISPOSABLE_RUNNER=1, run arm_worker.py execute with the envelope mode, and return artifacts only after the process exits.", + "family": "S1", + "manifest_sha256": "95ce8de8aca5580c8be95613b6058baecf2d473d9241831657e2a939577919c9" +} diff --git a/experiments/situated-experience-benchmark-v1/execution/substrate-preflight.json b/experiments/situated-experience-benchmark-v1/execution/substrate-preflight.json new file mode 100644 index 0000000..18c59c1 --- /dev/null +++ b/experiments/situated-experience-benchmark-v1/execution/substrate-preflight.json @@ -0,0 +1,122 @@ +{ + "schema_version": "1.0.0", + "benchmark_id": "situated-experience-benchmark-v1", + "family": "S1", + "recorded_at": "2026-08-14T21:04:18Z", + "implementation_commit": "be072873311621bfc7f56606db70b2f8a40d5bb5", + "manifest_sha256": "95ce8de8aca5580c8be95613b6058baecf2d473d9241831657e2a939577919c9", + "execution_plan_sha256": "6e6a3b75102d03d804cf0b8e1f51b3b1194fe5e1c39802b9d0cc64043bb9582a", + "status": "blocked", + "blocker_code": "hosted_model_credential_and_encryption_recipient_unavailable", + "hosted_evidence": { + "workflow_run_id": 31840751530, + "workflow_job_id": 94896861841, + "run_url": "https://github.com/yao23/agent-experience-graph/actions/runs/31840751530", + "job_url": "https://github.com/yao23/agent-experience-graph/actions/runs/31840751530/job/94896861841", + "sanitized_artifact_id": 9234146186, + "sanitized_artifact_sha256": "3a515538ae715d82a3fee624a4a838b1a3ca0bdbe2eaa6fc65ab1bf90c2465ae" + }, + "runner": { + "label": "ubuntu-24.04", + "image_os": "ubuntu24", + "image_version": "20260810.271.1" + }, + "container": { + "base_image": "python:3.12.11-slim-bookworm@sha256:519591d6871b7bc437060736b9f7456b8731f1499a57e22e6c285135ae657bf7", + "runtime_image_id": "sha256:423c7064cc5a754bec9c1a40756a27bd1814f0ed428b6de68250bfbd6fe9f005", + "security_configuration_matches": true, + "host_bind_mounts": 0, + "workspace_storage": "size-limited-tmpfs" + }, + "fixture_revalidation": { + "buggy_failures_matched": 4, + "human_patches_and_registered_suites_passed": 4, + "details": [ + "s1-01-scrapy-cookiejar source", + "s1-01-scrapy-cookiejar transfer", + "s1-02-fastapi-pydantic source", + "s1-02-fastapi-pydantic transfer" + ] + }, + "credential_boundary": { + "credential_present_in_controller": false, + "credential_present_in_repair": false, + "github_token_present_in_repair": false, + "configured_actions_secret_names": [], + "evidence_policy": "No credential value or credential hash was read or recorded." + }, + "canary": { + "status": "blocked", + "attempts": 29, + "passed": 28, + "failed": 1, + "passed_attempt_ids": [ + "controller_files", + "api_key", + "github_token", + "environment_allowlist", + "other_arm", + "docker_socket", + "network_tcp", + "network_dns", + "hidden_tests", + "human_patch", + "prior_cache_or_transcript", + "prior_model_conversation", + "symlink_escape", + "absolute_path_escape", + "proc_escape", + "subprocess_inheritance", + "container_security_configuration", + "process_limit", + "memory_limit", + "workspace_tmpfs_limit", + "wall_time_limit", + "command_limit", + "token_limit", + "cost_limit", + "disk_limit", + "patch_export_allowlist", + "repair_termination_before_evaluator", + "sanitizer_measurement_schema" + ], + "failed_attempt": { + "id": "model_or_encryption_boundary", + "reason": "host model credential is unavailable; raw-output public certificate is unavailable" + }, + "plaintext_raw_output_removed": true, + "benchmark_arms_executed": 0 + }, + "model_and_cost": { + "model": "gpt-5.6-sol", + "model_calls": 0, + "control_token_telemetry": false, + "treatment_token_telemetry": false, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "reason": "The host model credential was absent, so no model request was attempted." + }, + "requirements": [ + {"id": "fresh_disposable_runner_per_arm", "status": "implemented", "evidence": "The frozen matrix assigns every coordinate to a distinct GitHub-hosted job."}, + {"id": "exactly_one_sanitized_envelope_per_runner", "status": "passed", "evidence": "The controller accepts only arm.json plus task and streams that checked bundle into an empty tmpfs."}, + {"id": "cross_arm_isolation", "status": "passed", "evidence": "The hosted other-arm sentinel was unreachable; arm jobs use fresh VMs and no shared artifact input."}, + {"id": "controller_checkout_isolation", "status": "passed", "evidence": "The repair container has no host bind mount and the controller sentinel was unreachable."}, + {"id": "hidden_evaluator_isolation", "status": "passed", "evidence": "Hidden files were absent before repair termination and present only in the distinct evaluator canary."}, + {"id": "human_patch_and_future_history_isolation", "status": "passed", "evidence": "Human patches were absent and task packaging retains only the frozen one-commit seed with no remote."}, + {"id": "model_credential_brokering", "status": "blocked", "evidence": "The repair boundary proves credentials are absent, but no OPENAI_API_KEY is configured for the host controller."}, + {"id": "cache_log_conversation_workspace_and_process_isolation", "status": "passed", "evidence": "Hosted cache, transcript, conversation, sibling, namespace, and subprocess probes passed."}, + {"id": "evaluator_after_agent_termination", "status": "passed", "evidence": "The hosted repair container was absent before the distinct evaluator container started."}, + {"id": "private_raw_output_retention_and_sanitized_return", "status": "blocked", "evidence": "Sanitization passed and plaintext was removed; encryption could not run without AEG_RAW_OUTPUT_CERT_PEM."}, + {"id": "token_telemetry_for_both_modes", "status": "blocked", "evidence": "Unit accounting passed, but live control and treatment calls were prohibited by the absent credential."} + ], + "arms": { + "planned": 12, + "started": 0, + "completed": 0, + "task_failed": 0, + "infrastructure_failed": 0 + }, + "stop_action": "Keep the S1 execution matrix disabled until both Actions secrets are configured and this same hosted canary passes.", + "classification": "infrastructure-blocked" +} diff --git a/experiments/situated-experience-benchmark-v1/fixtures/s1-01-scrapy-cookiejar/PROVENANCE.md b/experiments/situated-experience-benchmark-v1/fixtures/s1-01-scrapy-cookiejar/PROVENANCE.md new file mode 100644 index 0000000..562ebbe --- /dev/null +++ b/experiments/situated-experience-benchmark-v1/fixtures/s1-01-scrapy-cookiejar/PROVENANCE.md @@ -0,0 +1,11 @@ +# Public provenance + +- Project: Scrapy (`scrapy/scrapy`), BSD-3-Clause. +- Source bug: BugsInPy `scrapy-31`; buggy `5f02ef82e8560242eb34b336f385addfdef3211d`; human fix `dba7e39f61cbe2c22d3c9064f32f6e36d74f14b2` (2015-08-03). +- Transfer bug: BugsInPy `scrapy-19`; buggy `e328a9b9dfa4fbc79c59ed4f45f757e998301c31`; human fix `1f743996ff00a7b728d59b93d0967e1eb50072f0` (2016-02-07). +- Upstream evidence: and . + +The staged files are dependency-free extracts of the named public production +methods and regression mechanisms. Names and behavioral assertions are retained +where needed for auditability; unrelated Scrapy code and dependencies are not +included. The controller validates both buggy failures and both human repairs. diff --git a/experiments/situated-experience-benchmark-v1/fixtures/s1-01-scrapy-cookiejar/experience.json b/experiments/situated-experience-benchmark-v1/fixtures/s1-01-scrapy-cookiejar/experience.json new file mode 100644 index 0000000..3fc5831 --- /dev/null +++ b/experiments/situated-experience-benchmark-v1/fixtures/s1-01-scrapy-cookiejar/experience.json @@ -0,0 +1,10 @@ +{ + "context": "A framework request adapter sits between Scrapy's request model and the Python standard-library CookieJar protocol.", + "version_constraints": "The source is Scrapy's Python 3 path after urllib/cookiejar began consuming native text and request attributes differently from Python 2; confirm the active Python 3 CookieJar call surface.", + "failed_approach": "Strict default UTF-8 conversion inside the adapter treated the boundary as ordinary application text and allowed one non-UTF-8 header to abort cookie parsing.", + "invalidating_evidence": "The failure persisted while core request data remained correct, and changing the core Request model or the standard library would widen the repair beyond the compatibility boundary.", + "recovery_principle": "Treat the adapter as an explicit versioned protocol boundary: enumerate the dependency's required surface and implement tolerant, local translations without changing the underlying request model.", + "validated_outcome": "The source human repair made all adapter header conversions tolerant while preserving ordinary CookieJar behavior and the surrounding request contract.", + "applicability_conditions": "Use only when local evidence shows a Python 3 standard-library consumer calling a partially implemented Scrapy adapter and the equivalent values already exist behind older methods.", + "known_invalidation_conditions": "Do not apply when failure originates in cookie policy, URL parsing, the core Request model, or a Python/runtime version whose CookieJar contract does not require the observed member." +} diff --git a/experiments/situated-experience-benchmark-v1/fixtures/s1-01-scrapy-cookiejar/source/buggy/cookie_adapter.py b/experiments/situated-experience-benchmark-v1/fixtures/s1-01-scrapy-cookiejar/source/buggy/cookie_adapter.py new file mode 100644 index 0000000..8cbb8ec --- /dev/null +++ b/experiments/situated-experience-benchmark-v1/fixtures/s1-01-scrapy-cookiejar/source/buggy/cookie_adapter.py @@ -0,0 +1,44 @@ +"""Dependency-free extract of Scrapy's pre-fix CookieJar adapter boundary.""" + + +def to_native_str(value, encoding="utf-8", errors="strict"): + if isinstance(value, bytes): + return value.decode(encoding, errors) + return str(value) + + +class Headers: + def __init__(self, values): + self._values = values + + def get(self, name, default=None): + values = self._values.get(name) + return values[0] if values else default + + def getlist(self, name): + return self._values.get(name, []) + + def items(self): + return self._values.items() + + +class WrappedRequest: + def __init__(self, headers): + self.headers = headers + + def get_header(self, name, default=None): + return to_native_str(self.headers.get(name, default)) + + def header_items(self): + return [ + (to_native_str(key), [to_native_str(value) for value in values]) + for key, values in self.headers.items() + ] + + +class WrappedResponse: + def __init__(self, headers): + self.headers = headers + + def get_all(self, name, default=None): + return [to_native_str(value) for value in self.headers.getlist(name)] diff --git a/experiments/situated-experience-benchmark-v1/fixtures/s1-01-scrapy-cookiejar/source/buggy/test_public.py b/experiments/situated-experience-benchmark-v1/fixtures/s1-01-scrapy-cookiejar/source/buggy/test_public.py new file mode 100644 index 0000000..47409fe --- /dev/null +++ b/experiments/situated-experience-benchmark-v1/fixtures/s1-01-scrapy-cookiejar/source/buggy/test_public.py @@ -0,0 +1,13 @@ +import unittest + +from cookie_adapter import Headers, WrappedResponse + + +class SourceMigrationTest(unittest.TestCase): + def test_non_utf8_cookie_header_does_not_abort_parsing(self): + wrapped = WrappedResponse(Headers({"Set-Cookie": [b"C1=in\xa3valid; path=/"]})) + self.assertEqual(wrapped.get_all("Set-Cookie"), ["C1=in�valid; path=/"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/experiments/situated-experience-benchmark-v1/fixtures/s1-01-scrapy-cookiejar/source/evaluator/human.patch b/experiments/situated-experience-benchmark-v1/fixtures/s1-01-scrapy-cookiejar/source/evaluator/human.patch new file mode 100644 index 0000000..4f7919d --- /dev/null +++ b/experiments/situated-experience-benchmark-v1/fixtures/s1-01-scrapy-cookiejar/source/evaluator/human.patch @@ -0,0 +1,25 @@ +diff --git a/cookie_adapter.py b/cookie_adapter.py +--- a/cookie_adapter.py ++++ b/cookie_adapter.py +@@ -31,11 +31,12 @@ class WrappedRequest: + self.headers = headers + + def get_header(self, name, default=None): +- return to_native_str(self.headers.get(name, default)) ++ return to_native_str(self.headers.get(name, default), errors="replace") + + def header_items(self): + return [ +- (to_native_str(key), [to_native_str(value) for value in values]) ++ (to_native_str(key, errors="replace"), ++ [to_native_str(value, errors="replace") for value in values]) + for key, values in self.headers.items() + ] + +@@ -45,4 +46,5 @@ class WrappedResponse: + self.headers = headers + + def get_all(self, name, default=None): +- return [to_native_str(value) for value in self.headers.getlist(name)] ++ return [to_native_str(value, errors="replace") ++ for value in self.headers.getlist(name)] diff --git a/experiments/situated-experience-benchmark-v1/fixtures/s1-01-scrapy-cookiejar/source/evaluator/test_hidden.py b/experiments/situated-experience-benchmark-v1/fixtures/s1-01-scrapy-cookiejar/source/evaluator/test_hidden.py new file mode 100644 index 0000000..edc070a --- /dev/null +++ b/experiments/situated-experience-benchmark-v1/fixtures/s1-01-scrapy-cookiejar/source/evaluator/test_hidden.py @@ -0,0 +1,19 @@ +import unittest + +from cookie_adapter import Headers, WrappedRequest + + +class SourceMigrationHiddenTest(unittest.TestCase): + def test_all_request_header_conversions_are_tolerant(self): + wrapped = WrappedRequest( + Headers({b"Other\xa3": [b"ignore\xa3me"], "Accept": [b"text/plain"]}) + ) + self.assertEqual(wrapped.get_header("Accept"), "text/plain") + self.assertEqual( + wrapped.header_items(), + [("Other�", ["ignore�me"]), ("Accept", ["text/plain"])], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/experiments/situated-experience-benchmark-v1/fixtures/s1-01-scrapy-cookiejar/transfer/agent/ISSUE.md b/experiments/situated-experience-benchmark-v1/fixtures/s1-01-scrapy-cookiejar/transfer/agent/ISSUE.md new file mode 100644 index 0000000..4a0cc73 --- /dev/null +++ b/experiments/situated-experience-benchmark-v1/fixtures/s1-01-scrapy-cookiejar/transfer/agent/ISSUE.md @@ -0,0 +1,10 @@ +# Python 3 CookieJar adapter migration + +The standard-library CookieJar contract changed on Python 3: it reads request +state through attributes that native urllib request objects expose. This +adapter still exposes only the older method-shaped interface, so cookie +processing raises `AttributeError` even though the equivalent values exist. + +Reproduce the public failure, make the smallest production-only repair in +`cookie_adapter.py`, and run the public test. Do not modify tests or inspect +outside this repository. diff --git a/experiments/situated-experience-benchmark-v1/fixtures/s1-01-scrapy-cookiejar/transfer/agent/cookie_adapter.py b/experiments/situated-experience-benchmark-v1/fixtures/s1-01-scrapy-cookiejar/transfer/agent/cookie_adapter.py new file mode 100644 index 0000000..ed31bf9 --- /dev/null +++ b/experiments/situated-experience-benchmark-v1/fixtures/s1-01-scrapy-cookiejar/transfer/agent/cookie_adapter.py @@ -0,0 +1,37 @@ +"""Dependency-free extract of Scrapy's Python 3 CookieJar request adapter.""" + +from urllib.parse import urlparse + + +class Request: + def __init__(self, url, meta=None, headers=None): + self.url = url + self.meta = meta or {} + self.headers = headers or {} + + +class WrappedRequest: + def __init__(self, request): + self.request = request + + def get_full_url(self): + return self.request.url + + def get_host(self): + return urlparse(self.request.url).netloc + + def get_type(self): + return urlparse(self.request.url).scheme + + def is_unverifiable(self): + return self.request.meta.get("is_unverifiable", False) + + @property + def unverifiable(self): + return self.is_unverifiable() + + def get_origin_req_host(self): + return urlparse(self.request.url).hostname + + def has_header(self, name): + return name in self.request.headers diff --git a/experiments/situated-experience-benchmark-v1/fixtures/s1-01-scrapy-cookiejar/transfer/agent/test_public.py b/experiments/situated-experience-benchmark-v1/fixtures/s1-01-scrapy-cookiejar/transfer/agent/test_public.py new file mode 100644 index 0000000..3aa2ab0 --- /dev/null +++ b/experiments/situated-experience-benchmark-v1/fixtures/s1-01-scrapy-cookiejar/transfer/agent/test_public.py @@ -0,0 +1,13 @@ +import unittest + +from cookie_adapter import Request, WrappedRequest + + +class CookieJarPython3ContractTest(unittest.TestCase): + def test_python3_cookiejar_reads_full_url_as_attribute(self): + wrapped = WrappedRequest(Request("https://www.example.com/path")) + self.assertEqual(wrapped.full_url, "https://www.example.com/path") + + +if __name__ == "__main__": + unittest.main() diff --git a/experiments/situated-experience-benchmark-v1/fixtures/s1-01-scrapy-cookiejar/transfer/evaluator/human.patch b/experiments/situated-experience-benchmark-v1/fixtures/s1-01-scrapy-cookiejar/transfer/evaluator/human.patch new file mode 100644 index 0000000..b8ff085 --- /dev/null +++ b/experiments/situated-experience-benchmark-v1/fixtures/s1-01-scrapy-cookiejar/transfer/evaluator/human.patch @@ -0,0 +1,32 @@ +diff --git a/cookie_adapter.py b/cookie_adapter.py +--- a/cookie_adapter.py ++++ b/cookie_adapter.py +@@ -23,6 +23,18 @@ class WrappedRequest: + def is_unverifiable(self): + return self.request.meta.get("is_unverifiable", False) + ++ @property ++ def full_url(self): ++ return self.get_full_url() ++ ++ @property ++ def host(self): ++ return self.get_host() ++ ++ @property ++ def type(self): ++ return self.get_type() ++ + @property + def unverifiable(self): + return self.is_unverifiable() +@@ -30,5 +42,9 @@ class WrappedRequest: + def get_origin_req_host(self): + return urlparse(self.request.url).hostname + ++ @property ++ def origin_req_host(self): ++ return self.get_origin_req_host() ++ + def has_header(self, name): + return name in self.request.headers diff --git a/experiments/situated-experience-benchmark-v1/fixtures/s1-01-scrapy-cookiejar/transfer/evaluator/test_hidden.py b/experiments/situated-experience-benchmark-v1/fixtures/s1-01-scrapy-cookiejar/transfer/evaluator/test_hidden.py new file mode 100644 index 0000000..f7d4075 --- /dev/null +++ b/experiments/situated-experience-benchmark-v1/fixtures/s1-01-scrapy-cookiejar/transfer/evaluator/test_hidden.py @@ -0,0 +1,29 @@ +import unittest + +from cookie_adapter import Request, WrappedRequest + + +class CookieJarPython3HiddenContractTest(unittest.TestCase): + def setUp(self): + self.wrapped = WrappedRequest( + Request( + "https://www.example.com/path", + meta={"is_unverifiable": True}, + headers={"content-type": "text/plain"}, + ) + ) + + def test_remaining_python3_cookiejar_attributes(self): + self.assertEqual(self.wrapped.host, "www.example.com") + self.assertEqual(self.wrapped.type, "https") + self.assertEqual(self.wrapped.origin_req_host, "www.example.com") + self.assertTrue(self.wrapped.unverifiable) + + def test_legacy_methods_and_header_contract_remain_valid(self): + self.assertEqual(self.wrapped.get_host(), "www.example.com") + self.assertEqual(self.wrapped.get_type(), "https") + self.assertTrue(self.wrapped.has_header("content-type")) + + +if __name__ == "__main__": + unittest.main() diff --git a/experiments/situated-experience-benchmark-v1/fixtures/s1-02-fastapi-pydantic/PROVENANCE.md b/experiments/situated-experience-benchmark-v1/fixtures/s1-02-fastapi-pydantic/PROVENANCE.md new file mode 100644 index 0000000..17e9bed --- /dev/null +++ b/experiments/situated-experience-benchmark-v1/fixtures/s1-02-fastapi-pydantic/PROVENANCE.md @@ -0,0 +1,12 @@ +# Public provenance + +- Project: FastAPI (`fastapi/fastapi`), MIT. +- Source bug: BugsInPy `fastapi-11`; buggy `bf229ad5d830eb5320f966d51a55e590e8d57008`; human fix `06eb4219345a77d23484528c9d164eb8d2097fec` (2019-08-07). +- Transfer bug: BugsInPy `fastapi-6`; buggy `5db99a27cf640864b4793807811848698c5ff4a2`; human fix `874d24181e779ebc6e1c52afb7d6598f863fd6a8` (2020-01-17). +- Upstream evidence: and . + +The staged files are dependency-free extracts of the named public Pydantic +classification and form-extraction mechanisms. They retain the version-shaped +metadata distinction and the upstream repair semantics while excluding +unrelated framework code and historical dependencies. The controller validates +both buggy failures and both human repairs. diff --git a/experiments/situated-experience-benchmark-v1/fixtures/s1-02-fastapi-pydantic/experience.json b/experiments/situated-experience-benchmark-v1/fixtures/s1-02-fastapi-pydantic/experience.json new file mode 100644 index 0000000..8dabb65 --- /dev/null +++ b/experiments/situated-experience-benchmark-v1/fixtures/s1-02-fastapi-pydantic/experience.json @@ -0,0 +1,10 @@ +{ + "context": "FastAPI classifies and extracts Pydantic request fields at a dependency boundary where equivalent annotations can appear in several metadata representations.", + "version_constraints": "The source and transfer use Pydantic 1.x-era Field metadata; shape, concrete type, and sub-fields vary with Python annotation syntax and Pydantic version.", + "failed_approach": "Inspecting only the top-level field shape classified a composite field as scalar even when nested metadata carried model or sequence semantics.", + "invalidating_evidence": "The wrong extraction/classification persisted while submitted form data was complete, showing that multipart parsing was not the failing layer and a global parser repair was unsupported.", + "recovery_principle": "At the Pydantic integration boundary, inspect every dependency representation that can encode the invariant—shape, concrete type, and sub-fields—then preserve existing scalar and sequence paths.", + "validated_outcome": "The source human repair recursively classified Union sub-fields and restored correct body handling without weakening scalar fields.", + "applicability_conditions": "Use when local evidence confirms Pydantic represents an equivalent collection or composite annotation outside the single metadata field the current predicate inspects.", + "known_invalidation_conditions": "Do not apply to raw multipart parsing failures, Pydantic 2 field APIs, response serialization, or cases where shape and concrete type already agree." +} diff --git a/experiments/situated-experience-benchmark-v1/fixtures/s1-02-fastapi-pydantic/source/buggy/field_classifier.py b/experiments/situated-experience-benchmark-v1/fixtures/s1-02-fastapi-pydantic/source/buggy/field_classifier.py new file mode 100644 index 0000000..c8d459c --- /dev/null +++ b/experiments/situated-experience-benchmark-v1/fixtures/s1-02-fastapi-pydantic/source/buggy/field_classifier.py @@ -0,0 +1,32 @@ +"""Dependency-free extract of FastAPI's Pydantic field classifier.""" + +from dataclasses import dataclass, field as dataclass_field + + +SINGLETON = "singleton" +SEQUENCE_TYPES = (list, set, tuple, dict) + + +class BaseModel: + pass + + +class Body: + pass + + +@dataclass +class Field: + shape: str = SINGLETON + type_: type = str + schema: object = None + sub_fields: list = dataclass_field(default_factory=list) + + +def is_scalar_field(field): + return ( + field.shape == SINGLETON + and not issubclass(field.type_, BaseModel) + and not issubclass(field.type_, SEQUENCE_TYPES) + and not isinstance(field.schema, Body) + ) diff --git a/experiments/situated-experience-benchmark-v1/fixtures/s1-02-fastapi-pydantic/source/buggy/test_public.py b/experiments/situated-experience-benchmark-v1/fixtures/s1-02-fastapi-pydantic/source/buggy/test_public.py new file mode 100644 index 0000000..c883ba0 --- /dev/null +++ b/experiments/situated-experience-benchmark-v1/fixtures/s1-02-fastapi-pydantic/source/buggy/test_public.py @@ -0,0 +1,17 @@ +import unittest + +from field_classifier import BaseModel, Field, is_scalar_field + + +class Item(BaseModel): + pass + + +class SourcePydanticRepresentationTest(unittest.TestCase): + def test_union_with_model_subfield_is_not_scalar(self): + union = Field(type_=object, sub_fields=[Field(type_=str), Field(type_=Item)]) + self.assertFalse(is_scalar_field(union)) + + +if __name__ == "__main__": + unittest.main() diff --git a/experiments/situated-experience-benchmark-v1/fixtures/s1-02-fastapi-pydantic/source/evaluator/human.patch b/experiments/situated-experience-benchmark-v1/fixtures/s1-02-fastapi-pydantic/source/evaluator/human.patch new file mode 100644 index 0000000..04ac8a7 --- /dev/null +++ b/experiments/situated-experience-benchmark-v1/fixtures/s1-02-fastapi-pydantic/source/evaluator/human.patch @@ -0,0 +1,20 @@ +diff --git a/field_classifier.py b/field_classifier.py +--- a/field_classifier.py ++++ b/field_classifier.py +@@ -24,9 +24,14 @@ class Field: + + + def is_scalar_field(field): +- return ( ++ if not ( + field.shape == SINGLETON + and not issubclass(field.type_, BaseModel) + and not issubclass(field.type_, SEQUENCE_TYPES) + and not isinstance(field.schema, Body) +- ) ++ ): ++ return False ++ if field.sub_fields: ++ if not all(is_scalar_field(sub_field) for sub_field in field.sub_fields): ++ return False ++ return True diff --git a/experiments/situated-experience-benchmark-v1/fixtures/s1-02-fastapi-pydantic/source/evaluator/test_hidden.py b/experiments/situated-experience-benchmark-v1/fixtures/s1-02-fastapi-pydantic/source/evaluator/test_hidden.py new file mode 100644 index 0000000..d82d472 --- /dev/null +++ b/experiments/situated-experience-benchmark-v1/fixtures/s1-02-fastapi-pydantic/source/evaluator/test_hidden.py @@ -0,0 +1,17 @@ +import unittest + +from field_classifier import Field, is_scalar_field + + +class SourcePydanticHiddenRepresentationTest(unittest.TestCase): + def test_union_with_sequence_subfield_is_not_scalar(self): + union = Field(type_=object, sub_fields=[Field(type_=str), Field(type_=list)]) + self.assertFalse(is_scalar_field(union)) + + def test_union_of_scalar_subfields_remains_scalar(self): + union = Field(type_=object, sub_fields=[Field(type_=str), Field(type_=int)]) + self.assertTrue(is_scalar_field(union)) + + +if __name__ == "__main__": + unittest.main() diff --git a/experiments/situated-experience-benchmark-v1/fixtures/s1-02-fastapi-pydantic/transfer/agent/ISSUE.md b/experiments/situated-experience-benchmark-v1/fixtures/s1-02-fastapi-pydantic/transfer/agent/ISSUE.md new file mode 100644 index 0000000..ff41f5a --- /dev/null +++ b/experiments/situated-experience-benchmark-v1/fixtures/s1-02-fastapi-pydantic/transfer/agent/ISSUE.md @@ -0,0 +1,10 @@ +# Pydantic collection representation migration + +FastAPI form extraction recognizes typing-based sequence fields but loses +repeated values when Pydantic represents an equivalent annotation as a concrete +built-in `list`, `set`, or `tuple` class. Under this dependency representation, +the field shape can remain scalar even though its concrete type is a sequence. + +Reproduce the public failure, preserve scalar and typing-based behavior, make +the smallest production-only repair in `form_extractor.py`, and run the public +test. Do not modify tests or inspect outside this repository. diff --git a/experiments/situated-experience-benchmark-v1/fixtures/s1-02-fastapi-pydantic/transfer/agent/form_extractor.py b/experiments/situated-experience-benchmark-v1/fixtures/s1-02-fastapi-pydantic/transfer/agent/form_extractor.py new file mode 100644 index 0000000..613ee37 --- /dev/null +++ b/experiments/situated-experience-benchmark-v1/fixtures/s1-02-fastapi-pydantic/transfer/agent/form_extractor.py @@ -0,0 +1,37 @@ +"""Dependency-free extract of FastAPI's form request-body extraction path.""" + +from dataclasses import dataclass + + +SEQUENCE_SHAPES = {"list-shape", "set-shape", "tuple-shape"} +SEQUENCE_TYPES = (list, set, tuple) + + +class FormData: + def __init__(self, pairs): + self._pairs = list(pairs) + + def get(self, alias): + values = [value for key, value in self._pairs if key == alias] + return values[-1] if values else None + + def getlist(self, alias): + return [value for key, value in self._pairs if key == alias] + + +@dataclass +class Field: + alias: str + shape: str + type_: type + + +def request_body_to_args(required_params, received_body): + values = {} + for field in required_params: + if field.shape in SEQUENCE_SHAPES and isinstance(received_body, FormData): + value = received_body.getlist(field.alias) + else: + value = received_body.get(field.alias) + values[field.alias] = value + return values diff --git a/experiments/situated-experience-benchmark-v1/fixtures/s1-02-fastapi-pydantic/transfer/agent/test_public.py b/experiments/situated-experience-benchmark-v1/fixtures/s1-02-fastapi-pydantic/transfer/agent/test_public.py new file mode 100644 index 0000000..4e17855 --- /dev/null +++ b/experiments/situated-experience-benchmark-v1/fixtures/s1-02-fastapi-pydantic/transfer/agent/test_public.py @@ -0,0 +1,17 @@ +import unittest + +from form_extractor import Field, FormData, request_body_to_args + + +class ConcreteSequenceFormTest(unittest.TestCase): + def test_builtin_list_receives_all_submitted_values(self): + form = FormData([("items", "first"), ("items", "second"), ("items", "third")]) + field = Field(alias="items", shape="singleton", type_=list) + self.assertEqual( + request_body_to_args([field], form)["items"], + ["first", "second", "third"], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/experiments/situated-experience-benchmark-v1/fixtures/s1-02-fastapi-pydantic/transfer/evaluator/human.patch b/experiments/situated-experience-benchmark-v1/fixtures/s1-02-fastapi-pydantic/transfer/evaluator/human.patch new file mode 100644 index 0000000..e9b9a89 --- /dev/null +++ b/experiments/situated-experience-benchmark-v1/fixtures/s1-02-fastapi-pydantic/transfer/evaluator/human.patch @@ -0,0 +1,14 @@ +diff --git a/form_extractor.py b/form_extractor.py +--- a/form_extractor.py ++++ b/form_extractor.py +@@ -29,7 +29,9 @@ class Field: + def request_body_to_args(required_params, received_body): + values = {} + for field in required_params: +- if field.shape in SEQUENCE_SHAPES and isinstance(received_body, FormData): ++ if ( ++ field.shape in SEQUENCE_SHAPES or field.type_ in SEQUENCE_TYPES ++ ) and isinstance(received_body, FormData): + value = received_body.getlist(field.alias) + else: + value = received_body.get(field.alias) diff --git a/experiments/situated-experience-benchmark-v1/fixtures/s1-02-fastapi-pydantic/transfer/evaluator/test_hidden.py b/experiments/situated-experience-benchmark-v1/fixtures/s1-02-fastapi-pydantic/transfer/evaluator/test_hidden.py new file mode 100644 index 0000000..855b09d --- /dev/null +++ b/experiments/situated-experience-benchmark-v1/fixtures/s1-02-fastapi-pydantic/transfer/evaluator/test_hidden.py @@ -0,0 +1,31 @@ +import unittest + +from form_extractor import Field, FormData, request_body_to_args + + +class ConcreteSequenceFormHiddenTest(unittest.TestCase): + def setUp(self): + self.form = FormData( + [("items", "first"), ("items", "second"), ("name", "Ada")] + ) + + def test_other_concrete_sequence_types_receive_all_values(self): + for concrete in (set, tuple): + with self.subTest(concrete=concrete.__name__): + field = Field(alias="items", shape="singleton", type_=concrete) + self.assertEqual( + request_body_to_args([field], self.form)["items"], + ["first", "second"], + ) + + def test_typing_shape_and_scalar_paths_remain_valid(self): + shaped = Field(alias="items", shape="list-shape", type_=str) + scalar = Field(alias="name", shape="singleton", type_=str) + self.assertEqual( + request_body_to_args([shaped, scalar], self.form), + {"items": ["first", "second"], "name": "Ada"}, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/experiments/situated-experience-benchmark-v1/freeze.json b/experiments/situated-experience-benchmark-v1/freeze.json new file mode 100644 index 0000000..554829e --- /dev/null +++ b/experiments/situated-experience-benchmark-v1/freeze.json @@ -0,0 +1,19 @@ +{ + "schema_version": "1.0.0", + "benchmark_id": "situated-experience-benchmark-v1", + "family": "S1", + "frozen_at": "2026-08-14T18:46:10Z", + "arms_executed_at_freeze": 0, + "protected_inputs": { + "manifest_sha256": "95ce8de8aca5580c8be95613b6058baecf2d473d9241831657e2a939577919c9", + "registry_sha256": "165767cde12f359b2bee9d43b6ea4ae3de9705d2a4d0f9773fec4f254ffce137", + "screening_sha256": "ced8f0eb25af8c05016a609d7645a246d26214cfd93d738c52720894d7ddd2a5", + "candidate_screening_contract_sha256": "575b2bfe6e9c9f0d1ed4a44c7a80eeb51735da67a3c819f576d29364cfffe88e", + "decision_ledger_sha256": "f0c0ac0a2b2f6485f074459e89c419efdb0423385925f5d23ed350f1112df509", + "measurement_contract_sha256": "6596698ee60d1d5b08de90eb3ad17386f7021a60f8f535570a2d55cf658b8324", + "controller_sha256": "1a06bb911c09823479d98bbc8d7bfa9aa77a99a78f43c98dbe02d7d5a9511116", + "worker_sha256": "2a9cdf2f3c0c9ea799eef6b31ffca5e911d746db1536841b4ebddcd8e0ed07af", + "schemas_tree_sha256": "31a435e4ded99b3e3179b6f072d8dc83d41070d7aa969d368fc580f41fe1362a", + "fixtures_tree_sha256": "0cd42cac9497c7e410b906a86108d4e21613ed8a6a25728994a4a1a43f22cddc" + } +} diff --git a/experiments/situated-experience-benchmark-v1/registry.json b/experiments/situated-experience-benchmark-v1/registry.json new file mode 100644 index 0000000..b80258e --- /dev/null +++ b/experiments/situated-experience-benchmark-v1/registry.json @@ -0,0 +1,57 @@ +{ + "schema_version": "1.0.0", + "benchmark_id": "situated-experience-benchmark-v1", + "name": "Situated Experience Benchmark v1", + "purpose": "Test whether AEG provides value when successful repair depends on situated knowledge about version state, execution environment, historical failures, cross-module consequences, multi-agent handoffs, and experience applicability.", + "families": [ + { + "order": 1, + "id": "S1", + "name": "dependency and version migration", + "status": "frozen_ready", + "situated_knowledge": ["version state", "dependency representation", "execution environment", "experience applicability"], + "screening_rules_path": "CANDIDATE-SCREENING.md#s1-dependency-and-version-migration", + "manifest_path": "s1-manifest.json" + }, + { + "order": 2, + "id": "S2", + "name": "CI and deployment failures", + "status": "design_only", + "situated_knowledge": ["runner image", "CI matrix", "deployment environment", "historical job failure"], + "screening_rules_path": "CANDIDATE-SCREENING.md#s2-ci-and-deployment-failures" + }, + { + "order": 3, + "id": "S3", + "name": "cross-module regressions", + "status": "design_only", + "situated_knowledge": ["module ownership", "downstream consumers", "cross-module consequences"], + "screening_rules_path": "CANDIDATE-SCREENING.md#s3-cross-module-regressions" + }, + { + "order": 4, + "id": "S4", + "name": "Planner-Coder-Tester-Reviewer collaboration", + "status": "design_only", + "situated_knowledge": ["role state", "handoff fidelity", "artifact provenance", "multi-agent coordination"], + "screening_rules_path": "CANDIDATE-SCREENING.md#s4-planner-coder-tester-reviewer-collaboration" + }, + { + "order": 5, + "id": "S5", + "name": "misleading repairs and repeated failure paths", + "status": "design_only", + "situated_knowledge": ["historical false positives", "invalidated repair paths", "negative evidence"], + "screening_rules_path": "CANDIDATE-SCREENING.md#s5-misleading-repairs-and-repeated-failure-paths" + }, + { + "order": 6, + "id": "S6", + "name": "experience invalidation under environment drift", + "status": "design_only", + "situated_knowledge": ["environment drift", "experience invalidation", "abstention", "version constraints"], + "screening_rules_path": "CANDIDATE-SCREENING.md#s6-experience-invalidation-under-environment-drift" + } + ] +} diff --git a/experiments/situated-experience-benchmark-v1/run_benchmark.py b/experiments/situated-experience-benchmark-v1/run_benchmark.py new file mode 100644 index 0000000..dc01514 --- /dev/null +++ b/experiments/situated-experience-benchmark-v1/run_benchmark.py @@ -0,0 +1,615 @@ +#!/usr/bin/env python3 +"""Validate, package, and evaluate Situated Experience Benchmark v1 S1.""" + +import argparse +import hashlib +import json +import os +import random +import re +import shlex +import shutil +import subprocess +import sys +import tempfile +from datetime import datetime, timezone +from pathlib import Path + +from jsonschema import Draft202012Validator, FormatChecker + + +HERE = Path(__file__).resolve().parent +MANIFEST = HERE / "s1-manifest.json" +FREEZE = HERE / "freeze.json" +REGISTRY = HERE / "registry.json" +SCREENING = HERE / "candidate-screening.json" +DECISIONS = HERE / "decision-ledger.jsonl" +SCHEMAS = HERE / "schemas" +FIXTURES = HERE / "fixtures" +MODES = ("control", "aeg-assisted") +EXPERIENCE_FIELDS = ( + "context", + "version_constraints", + "failed_approach", + "invalidating_evidence", + "recovery_principle", + "validated_outcome", + "applicability_conditions", + "known_invalidation_conditions", +) +EXPECTED_BUNDLE_ENTRIES = {"agent-result.schema.json", "arm.json", "arm_worker.py", "workspace"} +IGNORED_TREE_PARTS = {".git", "__pycache__", ".pytest_cache"} + + +class ProtocolError(RuntimeError): + pass + + +def load_json(path): + with Path(path).open(encoding="utf-8") as handle: + return json.load(handle) + + +def write_json(path, value): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + +def sha256_bytes(value): + return hashlib.sha256(value).hexdigest() + + +def sha256_file(path): + return sha256_bytes(Path(path).read_bytes()) + + +def canonical_sha256(value): + return sha256_bytes(json.dumps(value, sort_keys=True, separators=(",", ":")).encode()) + + +def tree_sha256(root): + root = Path(root) + digest = hashlib.sha256() + for path in sorted(root.rglob("*")): + relative = path.relative_to(root) + if not path.is_file() or any(part in IGNORED_TREE_PARTS for part in relative.parts): + continue + name = relative.as_posix().encode() + data = path.read_bytes() + digest.update(len(name).to_bytes(8, "big")) + digest.update(name) + digest.update(len(data).to_bytes(8, "big")) + digest.update(data) + return digest.hexdigest() + + +def run(args, cwd=None, timeout=120, env=None, input_text=None): + return subprocess.run( + args, + cwd=cwd, + env=env, + input=input_text, + text=True, + capture_output=True, + timeout=timeout, + check=False, + ) + + +def schema_validate(instance, schema_name, label): + schema = load_json(SCHEMAS / schema_name) + validator = Draft202012Validator(schema, format_checker=FormatChecker()) + errors = sorted(validator.iter_errors(instance), key=lambda error: list(error.path)) + if errors: + rendered = "; ".join(f"{'.'.join(map(str, error.path)) or ''}: {error.message}" for error in errors) + raise ProtocolError(f"{label} schema invalid: {rendered}") + + +def parse_time(value): + return datetime.fromisoformat(value.replace("Z", "+00:00")) + + +def expected_orders(manifest): + rng = random.Random(manifest["protocol"]["randomization_seed"]) + return { + pair["pair_id"]: [ + list(MODES) if rng.randrange(2) == 0 else list(reversed(MODES)) + for _ in range(manifest["protocol"]["replicates_per_arm"]) + ] + for pair in manifest["pairs"] + } + + +def fixture_root(pair): + return FIXTURES / pair["pair_id"] + + +def resolve_relative(path_text): + path = (HERE / path_text).resolve() + if HERE not in path.parents: + raise ProtocolError(f"path escapes benchmark directory: {path_text}") + return path + + +def patch_added_lines(patch_text): + return [line[1:] for line in patch_text.splitlines() if line.startswith("+") and not line.startswith("+++") and len(line[1:].strip()) >= 12] + + +def validate_decision_ledger(): + previous = None + count = 0 + for raw in DECISIONS.read_text(encoding="utf-8").splitlines(): + if not raw.strip(): + continue + event = json.loads(raw) + count += 1 + if event.get("sequence") != count: + raise ProtocolError("decision ledger sequence is not contiguous") + if event.get("previous_event_sha256") != previous: + raise ProtocolError("decision ledger previous hash is invalid") + claimed = event.get("event_sha256") + material = {key: value for key, value in event.items() if key != "event_sha256"} + actual = canonical_sha256(material) + if claimed != actual: + raise ProtocolError(f"decision ledger hash invalid at sequence {count}") + previous = claimed + if count < 4: + raise ProtocolError("decision ledger omits required design decisions") + return {"events": count, "head_sha256": previous} + + +def validate_freeze(manifest): + freeze = load_json(FREEZE) + expected = { + "manifest_sha256": sha256_file(MANIFEST), + "registry_sha256": sha256_file(REGISTRY), + "screening_sha256": sha256_file(SCREENING), + "candidate_screening_contract_sha256": sha256_file(HERE / "CANDIDATE-SCREENING.md"), + "decision_ledger_sha256": sha256_file(DECISIONS), + "measurement_contract_sha256": sha256_file(HERE / "MEASUREMENT-CONTRACT.md"), + "controller_sha256": sha256_file(HERE / "run_benchmark.py"), + "worker_sha256": sha256_file(HERE / "arm_worker.py"), + "schemas_tree_sha256": tree_sha256(SCHEMAS), + "fixtures_tree_sha256": tree_sha256(FIXTURES), + } + if freeze.get("schema_version") != "1.0.0" or freeze.get("benchmark_id") != manifest["benchmark_id"]: + raise ProtocolError("freeze record identity is invalid") + if freeze.get("protected_inputs") != expected: + raise ProtocolError("frozen controller input changed") + if freeze.get("arms_executed_at_freeze") != 0: + raise ProtocolError("freeze record must precede every arm") + return freeze + + +def validate(): + manifest = load_json(MANIFEST) + registry = load_json(REGISTRY) + screening = load_json(SCREENING) + schema_validate(registry, "benchmark-registry.schema.json", "benchmark registry") + schema_validate(screening, "candidate-screening.schema.json", "candidate screening") + schema_validate(manifest, "manifest.schema.json", "S1 manifest") + if manifest["protocol"]["arm_orders"] != expected_orders(manifest): + raise ProtocolError("frozen arm orders differ from the randomization seed") + accepted = [item for item in screening["candidates"] if item["status"] == "accepted"] + rejected = [item for item in screening["candidates"] if item["status"] == "rejected"] + if len(accepted) != 2 or {item["pair_id"] for item in accepted} != {pair["pair_id"] for pair in manifest["pairs"]}: + raise ProtocolError("S1 must contain exactly the two accepted screened pairs") + if len(rejected) != 5: + raise ProtocolError("the frozen S1 screening ledger must preserve all five rejected candidates") + seen = set() + for pair in manifest["pairs"]: + pair_id = pair["pair_id"] + if pair_id in seen: + raise ProtocolError(f"duplicate pair: {pair_id}") + seen.add(pair_id) + if parse_time(pair["source"]["fixed_at"]) >= parse_time(pair["transfer"]["fixed_at"]): + raise ProtocolError(f"{pair_id}: source fix must predate transfer fix") + if pair["source"]["fixed_commit"] == pair["transfer"]["fixed_commit"]: + raise ProtocolError(f"{pair_id}: source and transfer fixes must differ") + root = fixture_root(pair) + experience_path = resolve_relative(pair["experience_path"]) + experience = load_json(experience_path) + schema_validate(experience, "experience.schema.json", f"{pair_id} experience") + if tuple(experience) != EXPERIENCE_FIELDS: + raise ProtocolError(f"{pair_id}: experience field order or allowlist changed") + paths_and_hashes = ( + (root / "source" / "buggy", pair["source_fixture_sha256"], True), + (root / "source" / "evaluator" / "human.patch", pair["source_human_patch_sha256"], False), + (root / "source" / "evaluator" / "test_hidden.py", pair["source_hidden_tests_sha256"], False), + (root / "transfer" / "agent", pair["agent_fixture_sha256"], True), + (root / "transfer" / "evaluator" / "human.patch", pair["human_patch_sha256"], False), + (root / "transfer" / "evaluator" / "test_hidden.py", pair["hidden_tests_sha256"], False), + (experience_path, pair["experience_sha256"], False), + ) + for path, expected, is_tree in paths_and_hashes: + if not path.exists(): + raise ProtocolError(f"{pair_id}: missing frozen fixture {path}") + actual = tree_sha256(path) if is_tree else sha256_file(path) + if actual != expected: + raise ProtocolError(f"{pair_id}: fixture hash changed for {path.relative_to(HERE)}") + for pattern in (pair["source_initial_failure_pattern"], pair["initial_failure_pattern"]): + re.compile(pattern, re.IGNORECASE | re.DOTALL) + for historical in pair["historical_failure_paths"]: + re.compile(historical["pattern"], re.IGNORECASE | re.DOTALL) + serialized = json.dumps(experience, sort_keys=True) + transfer_patch = (root / "transfer" / "evaluator" / "human.patch").read_text(encoding="utf-8") + forbidden = [pair["transfer"]["fixed_commit"], pair["human_patch_sha256"]] + patch_added_lines(transfer_patch) + for needle in forbidden: + if needle and needle in serialized: + raise ProtocolError(f"{pair_id}: experience reveals transfer patch material") + ledger = validate_decision_ledger() + freeze = validate_freeze(manifest) + return { + "benchmark_id": manifest["benchmark_id"], + "family": "S1", + "accepted_pairs": 2, + "rejected_candidates": len(rejected), + "planned_arms": 12, + "manifest_sha256": sha256_file(MANIFEST), + "decision_ledger": ledger, + "frozen_at": freeze["frozen_at"], + "status": "valid-frozen-before-execution", + } + + +def git_init(workspace): + steps = ( + ["git", "init", "-q"], + ["git", "add", "."], + ["git", "-c", "user.name=SEB Controller", "-c", "user.email=seb@example.invalid", "commit", "-qm", "frozen transfer seed"], + ) + for step in steps: + result = run(step, cwd=workspace) + if result.returncode: + raise ProtocolError(result.stderr) + + +def run_test(command_text, workspace): + result = run(shlex.split(command_text), cwd=workspace, timeout=120) + return { + "command": command_text, + "passed": result.returncode == 0, + "exit_code": result.returncode, + "stdout": result.stdout[-3000:], + "stderr": result.stderr[-3000:], + } + + +def apply_patch(workspace, patch_path): + check = run(["git", "apply", "--check", str(patch_path)], cwd=workspace) + if check.returncode: + raise ProtocolError(f"human patch is not applicable: {check.stderr}") + applied = run(["git", "apply", str(patch_path)], cwd=workspace) + if applied.returncode: + raise ProtocolError(f"human patch failed: {applied.stderr}") + + +def preflight_fixture(pair, stage): + root = fixture_root(pair) / stage + seed = root / ("buggy" if stage == "source" else "agent") + pattern = pair["source_initial_failure_pattern"] if stage == "source" else pair["initial_failure_pattern"] + command_text = pair["source_public_test_command"] if stage == "source" else pair["public_test_command"] + with tempfile.TemporaryDirectory(prefix=f"seb-{pair['pair_id']}-{stage}-") as raw: + workspace = Path(raw) / "workspace" + shutil.copytree(seed, workspace) + git_init(workspace) + initial = run_test(command_text, workspace) + if initial["passed"]: + raise ProtocolError(f"{pair['pair_id']} {stage}: buggy seed unexpectedly passed") + output = initial["stdout"] + "\n" + initial["stderr"] + if not re.search(pattern, output, re.IGNORECASE | re.DOTALL): + raise ProtocolError(f"{pair['pair_id']} {stage}: buggy seed failed for an unregistered reason") + apply_patch(workspace, root / "evaluator" / "human.patch") + changed = run(["git", "diff", "--name-only"], cwd=workspace).stdout.splitlines() + if any(name.startswith("test") or "/test" in name or name == "ISSUE.md" for name in changed): + raise ProtocolError(f"{pair['pair_id']} {stage}: human patch changes protected inputs") + shutil.copyfile(root / "evaluator" / "test_hidden.py", workspace / "test_hidden.py") + verified = run_test(pair["hidden_test_command"], workspace) + if not verified["passed"]: + raise ProtocolError(f"{pair['pair_id']} {stage}: human patch failed registered suites\n{verified['stdout']}\n{verified['stderr']}") + return {"initial": initial, "human_patch": verified, "changed_files": changed} + + +def walk_keys(value): + if isinstance(value, dict): + for key, child in value.items(): + yield key + yield from walk_keys(child) + elif isinstance(value, list): + for child in value: + yield from walk_keys(child) + + +def task_prompt(pair): + return (fixture_root(pair) / "transfer" / "agent" / "ISSUE.md").read_text(encoding="utf-8") + + +def find_pair(manifest, pair_id): + for pair in manifest["pairs"]: + if pair["pair_id"] == pair_id: + return pair + raise ProtocolError(f"unknown pair: {pair_id}") + + +def arm_id(pair_id, replicate, mode): + return f"{pair_id}--r{replicate:02d}--{mode}" + + +def package_arm(manifest, pair, replicate, mode, output): + if replicate not in (1, 2, 3) or mode not in MODES: + raise ProtocolError("invalid arm coordinate") + if output.exists(): + raise ProtocolError(f"output already exists: {output}") + output.mkdir(parents=True) + workspace = output / "workspace" + shutil.copytree(fixture_root(pair) / "transfer" / "agent", workspace) + git_init(workspace) + prompt = task_prompt(pair) + context_hash = sha256_bytes(b"") + envelope = { + "schema_version": "1.0.0", + "benchmark_id": manifest["benchmark_id"], + "family": "S1", + "arm_id": arm_id(pair["pair_id"], replicate, mode), + "pair_id": pair["pair_id"], + "replicate": replicate, + "mode": mode, + "order": manifest["protocol"]["arm_orders"][pair["pair_id"]][replicate - 1], + "model": manifest["protocol"]["model"], + "budget": manifest["protocol"]["budget"], + "task_prompt": prompt, + "public_test_command": pair["public_test_command"], + "allowed_experience_fields": list(EXPERIENCE_FIELDS), + "input_hashes": { + "manifest": sha256_file(MANIFEST), + "agent_fixture": pair["agent_fixture_sha256"], + "task_prompt": sha256_bytes(prompt.encode()), + "mode_context": context_hash, + }, + } + if mode == "aeg-assisted": + experience = load_json(resolve_relative(pair["experience_path"])) + envelope["experience_id"] = f"{pair['pair_id']}-source-experience" + envelope["experience"] = experience + envelope["input_hashes"]["mode_context"] = pair["experience_sha256"] + write_json(output / "arm.json", envelope) + shutil.copyfile(HERE / "arm_worker.py", output / "arm_worker.py") + shutil.copyfile(SCHEMAS / "agent-result.schema.json", output / "agent-result.schema.json") + audit_bundle(manifest, pair, output) + return envelope + + +def audit_bundle(manifest, pair, bundle): + actual = {path.name for path in bundle.iterdir()} + if actual != EXPECTED_BUNDLE_ENTRIES: + raise ProtocolError(f"bundle entries differ from allowlist: {sorted(actual ^ EXPECTED_BUNDLE_ENTRIES)}") + envelope = load_json(bundle / "arm.json") + forbidden_keys = {"source", "transfer", "fixed_commit", "human_patch", "hidden_tests", "evaluator", "historical_failure_paths"} + leaked_keys = forbidden_keys.intersection(walk_keys(envelope)) + if leaked_keys: + raise ProtocolError(f"bundle exposes controller/evaluator keys: {sorted(leaked_keys)}") + if envelope["mode"] == "control" and ("experience" in envelope or "experience_id" in envelope): + raise ProtocolError("control bundle contains experience data") + if envelope["mode"] == "aeg-assisted" and set(envelope.get("experience", {})) != set(EXPERIENCE_FIELDS): + raise ProtocolError("assisted payload differs from compact experience allowlist") + workspace = bundle / "workspace" + if run(["git", "remote"], cwd=workspace).stdout.strip(): + raise ProtocolError("bundle workspace has a remote") + if run(["git", "rev-list", "--all", "--count"], cwd=workspace).stdout.strip() != "1": + raise ProtocolError("bundle workspace does not have exactly one commit") + exposed_text = [(bundle / "arm.json").read_text(encoding="utf-8")] + for path in workspace.rglob("*"): + if path.is_file() and ".git" not in path.parts and path.stat().st_size <= 1_000_000: + exposed_text.append(path.read_text(encoding="utf-8", errors="replace")) + serialized = "\n".join(exposed_text) + root = fixture_root(pair) + transfer_patch = (root / "transfer" / "evaluator" / "human.patch").read_text(encoding="utf-8") + hidden = (root / "transfer" / "evaluator" / "test_hidden.py").read_text(encoding="utf-8") + forbidden_needles = [pair["transfer"]["fixed_commit"], pair["human_patch_sha256"], hidden] + patch_added_lines(transfer_patch) + forbidden_needles.extend(other["pair_id"] for other in manifest["pairs"] if other["pair_id"] != pair["pair_id"]) + for needle in forbidden_needles: + if needle and needle in serialized: + raise ProtocolError(f"bundle leaks evaluator or other-pair material: {needle[:60]!r}") + for forbidden_name in ("human.patch", "test_hidden.py", "evaluator.json", "prior-arm.patch", "prior-arm.log"): + if any(path.name == forbidden_name for path in workspace.rglob("*")): + raise ProtocolError(f"bundle workspace exposes {forbidden_name}") + return {"arm_id": envelope["arm_id"], "status": "passed"} + + +def preflight(): + summary = validate() + manifest = load_json(MANIFEST) + fixtures = {} + packaged = [] + for pair in manifest["pairs"]: + fixtures[pair["pair_id"]] = { + "source": preflight_fixture(pair, "source"), + "transfer": preflight_fixture(pair, "transfer"), + } + with tempfile.TemporaryDirectory(prefix="seb-package-preflight-") as raw: + root = Path(raw) + for pair in manifest["pairs"]: + for replicate in (1, 2, 3): + for mode in MODES: + output = root / arm_id(pair["pair_id"], replicate, mode) + envelope = package_arm(manifest, pair, replicate, mode, output) + packaged.append(envelope["arm_id"]) + summary.update({ + "fixture_preflights": 4, + "human_patches_verified": 4, + "arm_bundles_audited": len(packaged), + "checks": ["buggy-reason", "human-fix", "fixture-hash", "one-commit-arm", "control-separation", "experience-allowlist", "transfer-patch-leakage", "hidden-evaluator-access", "cross-pair-access"], + "status": "ready-no-arms-executed", + }) + return summary + + +def schedule_s1(output): + manifest = load_json(MANIFEST) + validate() + if output.exists(): + raise ProtocolError(f"output already exists: {output}") + output.mkdir(parents=True) + arms = [] + for pair in manifest["pairs"]: + for replicate, order in enumerate(manifest["protocol"]["arm_orders"][pair["pair_id"]], 1): + for mode in order: + target = output / "arms" / arm_id(pair["pair_id"], replicate, mode) + envelope = package_arm(manifest, pair, replicate, mode, target) + arms.append({"arm_id": envelope["arm_id"], "relative_bundle": str(target.relative_to(output)), "mode": mode}) + plan = { + "benchmark_id": manifest["benchmark_id"], + "family": "S1", + "manifest_sha256": sha256_file(MANIFEST), + "created_at": datetime.now(timezone.utc).isoformat(), + "arm_count": len(arms), + "arms": arms, + "execution_requirement": "Copy exactly one relative_bundle to each fresh disposable runner, set SEB_DISPOSABLE_RUNNER=1, run arm_worker.py execute with the envelope mode, and return artifacts only after the process exits.", + } + write_json(output / "execution-plan.json", plan) + return plan + + +def evaluate_arm(pair, replicate, mode, arm_output, destination): + manifest = load_json(MANIFEST) + validate() + record = load_json(arm_output / "arm-result.json") + if (record.get("pair_id"), record.get("replicate"), record.get("mode")) != (pair["pair_id"], replicate, mode): + raise ProtocolError("arm result coordinate differs from evaluator request") + patch = (arm_output / "patch.diff").read_text(encoding="utf-8") + with tempfile.TemporaryDirectory(prefix="seb-hidden-evaluator-") as raw: + workspace = Path(raw) / "workspace" + shutil.copytree(fixture_root(pair) / "transfer" / "agent", workspace) + git_init(workspace) + applied = run(["git", "apply", "--whitespace=nowarn", "-"], cwd=workspace, input_text=patch) + if applied.returncode: + public = {"passed": False, "command": pair["public_test_command"]} + hidden = {"passed": False, "command": pair["hidden_test_command"]} + changed = [] + findings = ["agent patch did not apply to the frozen seed"] + else: + changed = run(["git", "diff", "--name-only"], cwd=workspace).stdout.splitlines() + protected = any(path == "ISSUE.md" or path.startswith("test") or "/test" in path for path in changed) + public = run_test(pair["public_test_command"], workspace) + shutil.copyfile(fixture_root(pair) / "transfer" / "evaluator" / "test_hidden.py", workspace / "test_hidden.py") + hidden = run_test(pair["hidden_test_command"], workspace) + findings = [ + f"focused suite {'passed' if public['passed'] else 'failed'}", + f"hidden regression suite {'passed' if hidden['passed'] else 'failed'}", + f"protected inputs {'changed' if protected else 'unchanged'}", + ] + evidence = patch + "\n" + (arm_output / "events.jsonl").read_text(encoding="utf-8", errors="replace") + repeated = [item["id"] for item in pair["historical_failure_paths"] if re.search(item["pattern"], evidence, re.IGNORECASE | re.DOTALL)] + protected_changed = any(path == "ISSUE.md" or path.startswith("test") or "/test" in path for path in changed) + record["evaluation_status"] = "evaluated" + record["regression_free_success"] = bool(public["passed"] and hidden["passed"] and not protected_changed) + record["tests_run"].extend([ + {"command": pair["public_test_command"], "scope": "evaluator", "passed": bool(public["passed"])}, + {"command": pair["hidden_test_command"], "scope": "hidden", "passed": bool(hidden["passed"])}, + ]) + record["failed_historical_paths_repeated"] = repeated + reported_assumptions = { + item.get("assumption"): item + for item in record.get("environment_assumptions_checked", []) + if isinstance(item, dict) and item.get("assumption") + } + record["environment_assumptions_checked"] = [ + reported_assumptions.get( + assumption, + {"assumption": assumption, "checked": False, "evidence": "not reported by the agent"}, + ) + for assumption in pair["environment_assumptions"] + ] + record["evaluator_findings"] = findings + schema_validate(record, "arm-result.schema.json", "evaluated arm result") + write_json(destination, record) + return record + + +def compare_pair(control_path, treatment_path, output): + control = load_json(control_path) + treatment = load_json(treatment_path) + if control["evaluation_status"] != "evaluated" or treatment["evaluation_status"] != "evaluated": + raise ProtocolError("pair comparison requires two evaluated arms") + if (control["pair_id"], control["replicate"]) != (treatment["pair_id"], treatment["replicate"]): + raise ProtocolError("pair coordinates do not match") + if control["mode"] != "control" or treatment["mode"] != "aeg-assisted": + raise ProtocolError("pair modes are invalid") + thresholds = load_json(MANIFEST)["protocol"]["evaluation_thresholds"] + increase = 1 + thresholds["effort_change_fraction"] + measures = [ + (control["attempts"], treatment["attempts"]), + (control["completed_commands"], treatment["completed_commands"]), + (len(control["tests_run"]), len(treatment["tests_run"])), + (control["patch_size"]["added_lines"] + control["patch_size"]["deleted_lines"], treatment["patch_size"]["added_lines"] + treatment["patch_size"]["deleted_lines"]), + ] + control_tokens = control["tokens"]["input"] + control["tokens"]["output"] if control["tokens"]["input"] is not None and control["tokens"]["output"] is not None else None + treatment_tokens = treatment["tokens"]["input"] + treatment["tokens"]["output"] if treatment["tokens"]["input"] is not None and treatment["tokens"]["output"] is not None else None + if control_tokens is not None and treatment_tokens is not None: + measures.append((control_tokens, treatment_tokens)) + losses = sum(new > base * increase for base, new in measures if base > 0) + success_loss = control["regression_free_success"] and not treatment["regression_free_success"] + negative = bool(success_loss or (losses >= thresholds["negative_transfer_effort_measure_count"] and treatment["regression_free_success"] == control["regression_free_success"])) + control["negative_transfer"] = False + treatment["negative_transfer"] = negative + report = { + "pair_id": control["pair_id"], + "replicate": control["replicate"], + "control": control, + "treatment": treatment, + "negative_transfer": negative, + "stop_condition_triggered": "treatment_causes_additional_regressions" if success_loss else None, + } + write_json(output, report) + return report + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest="action", required=True) + sub.add_parser("validate") + sub.add_parser("preflight") + package = sub.add_parser("package-arm") + package.add_argument("--pair", required=True) + package.add_argument("--replicate", type=int, required=True) + package.add_argument("--mode", choices=MODES, required=True) + package.add_argument("--output", required=True) + schedule = sub.add_parser("schedule-s1") + schedule.add_argument("--output", required=True) + evaluate = sub.add_parser("evaluate-arm") + evaluate.add_argument("--pair", required=True) + evaluate.add_argument("--replicate", type=int, required=True) + evaluate.add_argument("--mode", choices=MODES, required=True) + evaluate.add_argument("--arm-output", required=True) + evaluate.add_argument("--output", required=True) + compare = sub.add_parser("compare-pair") + compare.add_argument("--control", required=True) + compare.add_argument("--treatment", required=True) + compare.add_argument("--output", required=True) + args = parser.parse_args() + if args.action == "validate": + result = validate() + elif args.action == "preflight": + result = preflight() + elif args.action == "schedule-s1": + result = schedule_s1(Path(args.output).resolve()) + elif args.action == "compare-pair": + result = compare_pair(Path(args.control), Path(args.treatment), Path(args.output)) + else: + manifest = load_json(MANIFEST) + pair = find_pair(manifest, args.pair) + if args.action == "package-arm": + validate() + result = package_arm(manifest, pair, args.replicate, args.mode, Path(args.output).resolve()) + else: + result = evaluate_arm(pair, args.replicate, args.mode, Path(args.arm_output).resolve(), Path(args.output).resolve()) + print(json.dumps(result, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except (ProtocolError, subprocess.TimeoutExpired, json.JSONDecodeError, re.error) as error: + print(f"situated benchmark protocol error: {error}", file=sys.stderr) + raise SystemExit(2) diff --git a/experiments/situated-experience-benchmark-v1/s1-manifest.json b/experiments/situated-experience-benchmark-v1/s1-manifest.json new file mode 100644 index 0000000..d8f01c4 --- /dev/null +++ b/experiments/situated-experience-benchmark-v1/s1-manifest.json @@ -0,0 +1,158 @@ +{ + "schema_version": "1.0.0", + "benchmark_id": "situated-experience-benchmark-v1", + "name": "Situated Experience Benchmark v1", + "status": "frozen-before-arm-execution", + "family": "S1", + "frozen_at": "2026-08-14T18:46:10Z", + "protocol": { + "replicates_per_arm": 3, + "modes": ["control", "aeg-assisted"], + "randomization_seed": "situated-experience-benchmark-v1-s1-2026-08-14", + "model": "gpt-5.6-sol", + "task_input_policy": "The transfer task, one-commit agent fixture, model, budget, and structured-result contract are byte-identical across modes. AEG-assisted receives only the frozen eight-field source experience; control receives no substitute advice.", + "budget": { + "wall_time_seconds": 900, + "max_completed_commands": 40, + "max_attempts": 3 + }, + "evaluation_thresholds": { + "effort_change_fraction": 0.2, + "max_treatment_token_overhead_fraction": 0.3, + "minimum_interpretable_repair_path_improvements": 1, + "negative_transfer_effort_measure_count": 2 + }, + "measurement_contract": "MEASUREMENT-CONTRACT.md", + "arm_orders": { + "s1-01-scrapy-cookiejar": [ + ["aeg-assisted", "control"], + ["aeg-assisted", "control"], + ["aeg-assisted", "control"] + ], + "s1-02-fastapi-pydantic": [ + ["control", "aeg-assisted"], + ["control", "aeg-assisted"], + ["control", "aeg-assisted"] + ] + } + }, + "promotion_criteria": [ + "no reduction in regression-free success", + "at least one interpretable repair-path improvement", + "no leakage or contamination", + "treatment token overhead no greater than 30% at the median", + "wall-time improvement alone cannot qualify as positive evidence" + ], + "stop_conditions": [ + "hidden evaluator data is reachable by an agent", + "one arm can read another arm's artifacts", + "treatment contains or reveals the transfer patch", + "task, thresholds, or metrics change after outcomes are observed", + "two accepted pairs are both trivial one-shot tasks with identical paths", + "treatment causes additional regressions" + ], + "pairs": [ + { + "pair_id": "s1-01-scrapy-cookiejar", + "title": "Scrapy CookieJar adapter across Python 3 protocol changes", + "project": "scrapy", + "license": "BSD-3-Clause", + "fixture_kind": "dependency-free-public-extract", + "source": { + "bug_id": "BugsInPy scrapy-31", + "buggy_commit": "5f02ef82e8560242eb34b336f385addfdef3211d", + "fixed_commit": "dba7e39f61cbe2c22d3c9064f32f6e36d74f14b2", + "fixed_at": "2015-08-03T10:53:40-03:00", + "evidence_url": "https://github.com/scrapy/scrapy/commit/dba7e39f61cbe2c22d3c9064f32f6e36d74f14b2" + }, + "transfer": { + "bug_id": "BugsInPy scrapy-19", + "buggy_commit": "e328a9b9dfa4fbc79c59ed4f45f757e998301c31", + "fixed_commit": "1f743996ff00a7b728d59b93d0967e1eb50072f0", + "fixed_at": "2016-02-07T14:19:27+09:00", + "evidence_url": "https://github.com/scrapy/scrapy/commit/1f743996ff00a7b728d59b93d0967e1eb50072f0" + }, + "experience_path": "fixtures/s1-01-scrapy-cookiejar/experience.json", + "experience_sha256": "221248cb25176a82d1c225512fa975f2193211159e7c8f7fa99aa1ae48c29e9a", + "source_fixture_sha256": "98cc2b20f1d4442f784f9ea361663730c75545429c5a30fd071169ef641224e1", + "source_human_patch_sha256": "6245f6f03eb01ecadb7196ef78e4722e8e0d1504e6e47f26785e795f84e64cac", + "source_hidden_tests_sha256": "593092563f21a4f948b64743137d613bf7ddd0f82a99326b0575099b9798e7a5", + "agent_fixture_sha256": "a4fdf752291d311f49ab0f8dc4a5a9c1c183c13c77afb65fd39f13383e2b01c8", + "human_patch_sha256": "ee17730a3f55556209695c9313a94136de9dcd578540b3063203d973cde33010", + "hidden_tests_sha256": "cfa5201669d335e35fb7dd21d794000f0910206b09fc3e6ce9e8f7a3549af398", + "source_public_test_command": "python3 -m unittest -v test_public.py", + "public_test_command": "python3 -m unittest -v test_public.py", + "hidden_test_command": "python3 -m unittest -v test_public.py test_hidden.py", + "source_initial_failure_pattern": "UnicodeDecodeError", + "initial_failure_pattern": "AttributeError.*full_url", + "historical_failure_paths": [ + { + "id": "patch_standard_library_cookiejar", + "description": "Modify or monkeypatch the standard-library CookieJar instead of the adapter boundary.", + "pattern": "(?:http\\.cookiejar|cookielib).{0,80}(?:patch|setattr|monkey)" + }, + { + "id": "widen_core_request_model", + "description": "Add CookieJar-only compatibility members to the core Scrapy Request model.", + "pattern": "(?:class Request|core Request).{0,120}(?:full_url|origin_req_host)" + } + ], + "environment_assumptions": [ + "Python 3 CookieJar consumes attribute-shaped request protocol members.", + "The adapter's legacy methods already return the correct underlying values.", + "The failure is inside the adapter boundary rather than cookie policy or URL parsing." + ] + }, + { + "pair_id": "s1-02-fastapi-pydantic", + "title": "FastAPI request handling across Pydantic field representations", + "project": "fastapi", + "license": "MIT", + "fixture_kind": "dependency-free-public-extract", + "source": { + "bug_id": "BugsInPy fastapi-11", + "buggy_commit": "bf229ad5d830eb5320f966d51a55e590e8d57008", + "fixed_commit": "06eb4219345a77d23484528c9d164eb8d2097fec", + "fixed_at": "2019-08-07T13:55:33-05:00", + "evidence_url": "https://github.com/fastapi/fastapi/commit/06eb4219345a77d23484528c9d164eb8d2097fec" + }, + "transfer": { + "bug_id": "BugsInPy fastapi-6", + "buggy_commit": "5db99a27cf640864b4793807811848698c5ff4a2", + "fixed_commit": "874d24181e779ebc6e1c52afb7d6598f863fd6a8", + "fixed_at": "2020-01-17T12:45:55+01:00", + "evidence_url": "https://github.com/fastapi/fastapi/commit/874d24181e779ebc6e1c52afb7d6598f863fd6a8" + }, + "experience_path": "fixtures/s1-02-fastapi-pydantic/experience.json", + "experience_sha256": "3645fbb143a670e0ead1d4cbc540a865a50bf9ec8c91abb53a90582361e49d8a", + "source_fixture_sha256": "2c2b7c18668300a6cfcd1e39d7617a23242733dfdb799756a3c721de289a4976", + "source_human_patch_sha256": "4a82762478f56df4795c5ec2297675552cdd407091338fbd6b98cde032148f46", + "source_hidden_tests_sha256": "5f1bc4b426f5da56dcb50f6a7d0d8d3fc83f5db1a6ee117ca121876d5abf34a1", + "agent_fixture_sha256": "5046ac3d26ec6685e385191146b6b3f353f0fa787278f866fd5547a0344ff4f9", + "human_patch_sha256": "6f004132a1f1c8d5013290e61e6f915746e48f49901ba965ff00e99c6bbf27b9", + "hidden_tests_sha256": "1e5efeb50d47d2ccdb9f1731789d84cfc0cabeefeb9c25e899deeb3b0ff1a76f", + "source_public_test_command": "python3 -m unittest -v test_public.py", + "public_test_command": "python3 -m unittest -v test_public.py", + "hidden_test_command": "python3 -m unittest -v test_public.py test_hidden.py", + "source_initial_failure_pattern": "True is not false", + "initial_failure_pattern": "\\['first', 'second', 'third'\\]", + "historical_failure_paths": [ + { + "id": "patch_multipart_parser", + "description": "Change raw multipart or FormData parsing even though submitted values are already complete.", + "pattern": "(?:multipart|formparsers|FormData parser).{0,100}(?:patch|change|edit)" + }, + { + "id": "special_case_list_only", + "description": "Repair only built-in list while leaving equivalent set and tuple representations broken.", + "pattern": "field\\.type_\\s*(?:==|is)\\s*list" + } + ], + "environment_assumptions": [ + "Pydantic 1.x can encode equivalent collection annotations through shape or concrete type.", + "FormData already retains every repeated submitted value.", + "Scalar and typing-shape extraction behavior must remain unchanged." + ] + } + ] +} diff --git a/experiments/situated-experience-benchmark-v1/schemas/agent-result.schema.json b/experiments/situated-experience-benchmark-v1/schemas/agent-result.schema.json new file mode 100644 index 0000000..a46d7d7 --- /dev/null +++ b/experiments/situated-experience-benchmark-v1/schemas/agent-result.schema.json @@ -0,0 +1,33 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "additionalProperties": false, + "required": [ + "summary", + "first_repair_location", + "proposed_approach", + "experience_disposition", + "experience_reason", + "environment_assumptions_checked" + ], + "properties": { + "summary": {"type": "string"}, + "first_repair_location": {"type": "string"}, + "proposed_approach": {"type": "string"}, + "experience_disposition": {"enum": ["used", "rejected", "abstained"]}, + "experience_reason": {"type": "string"}, + "environment_assumptions_checked": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["assumption", "checked", "evidence"], + "properties": { + "assumption": {"type": "string"}, + "checked": {"type": "boolean"}, + "evidence": {"type": "string"} + } + } + } + } +} diff --git a/experiments/situated-experience-benchmark-v1/schemas/arm-result.schema.json b/experiments/situated-experience-benchmark-v1/schemas/arm-result.schema.json new file mode 100644 index 0000000..d1a9254 --- /dev/null +++ b/experiments/situated-experience-benchmark-v1/schemas/arm-result.schema.json @@ -0,0 +1,40 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://aeg.dev/schemas/situated-arm-result-v1.json", + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", "benchmark_id", "family", "pair_id", "replicate", "mode", "evaluation_status", + "input_hashes", "budget", "regression_free_success", "attempts", + "completed_commands", "tests_run", "files_inspected", "files_changed", + "patch_size", "wall_time_ms", "tokens", "failed_historical_paths_repeated", + "environment_assumptions_checked", "experiences", "negative_transfer", + "evaluator_findings", "limitations" + ], + "properties": { + "schema_version": {"const": "1.0.0"}, + "benchmark_id": {"const": "situated-experience-benchmark-v1"}, + "family": {"const": "S1"}, + "pair_id": {"type": "string"}, + "replicate": {"type": "integer", "minimum": 1, "maximum": 3}, + "mode": {"enum": ["control", "aeg-assisted"]}, + "evaluation_status": {"enum": ["captured", "evaluated"]}, + "input_hashes": {"type": "object", "additionalProperties": {"type": "string", "pattern": "^[0-9a-f]{64}$"}, "minProperties": 3}, + "budget": {"type": "object", "additionalProperties": false, "required": ["wall_time_seconds", "max_completed_commands", "max_attempts"], "properties": {"wall_time_seconds": {"type": "integer", "minimum": 1}, "max_completed_commands": {"type": "integer", "minimum": 1}, "max_attempts": {"type": "integer", "minimum": 1}}}, + "regression_free_success": {"type": ["boolean", "null"]}, + "attempts": {"type": "integer", "minimum": 0}, + "completed_commands": {"type": "integer", "minimum": 0}, + "tests_run": {"type": "array", "items": {"type": "object", "additionalProperties": false, "required": ["command", "scope", "passed"], "properties": {"command": {"type": "string"}, "scope": {"enum": ["agent", "focused", "hidden", "regression", "evaluator"]}, "passed": {"type": "boolean"}}}}, + "files_inspected": {"type": "array", "uniqueItems": true, "items": {"type": "string"}}, + "files_changed": {"type": "array", "uniqueItems": true, "items": {"type": "string"}}, + "patch_size": {"type": "object", "additionalProperties": false, "required": ["added_lines", "deleted_lines", "files"], "properties": {"added_lines": {"type": "integer", "minimum": 0}, "deleted_lines": {"type": "integer", "minimum": 0}, "files": {"type": "integer", "minimum": 0}}}, + "wall_time_ms": {"type": "integer", "minimum": 0}, + "tokens": {"type": "object", "additionalProperties": false, "required": ["input", "output", "unavailable_reason"], "properties": {"input": {"type": ["integer", "null"], "minimum": 0}, "output": {"type": ["integer", "null"], "minimum": 0}, "unavailable_reason": {"type": ["string", "null"]}}}, + "failed_historical_paths_repeated": {"type": "array", "uniqueItems": true, "items": {"type": "string"}}, + "environment_assumptions_checked": {"type": "array", "items": {"type": "object", "additionalProperties": false, "required": ["assumption", "checked", "evidence"], "properties": {"assumption": {"type": "string"}, "checked": {"type": "boolean"}, "evidence": {"type": "string"}}}}, + "experiences": {"type": "array", "items": {"type": "object", "additionalProperties": false, "required": ["experience_id", "disposition", "reason"], "properties": {"experience_id": {"type": ["string", "null"]}, "disposition": {"enum": ["retrieved", "used", "rejected", "abstained"]}, "reason": {"type": "string"}}}}, + "negative_transfer": {"type": ["boolean", "null"]}, + "evaluator_findings": {"type": "array", "items": {"type": "string"}}, + "limitations": {"type": "array", "items": {"type": "string"}} + } +} diff --git a/experiments/situated-experience-benchmark-v1/schemas/benchmark-registry.schema.json b/experiments/situated-experience-benchmark-v1/schemas/benchmark-registry.schema.json new file mode 100644 index 0000000..92662ab --- /dev/null +++ b/experiments/situated-experience-benchmark-v1/schemas/benchmark-registry.schema.json @@ -0,0 +1,50 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://aeg.dev/schemas/situated-experience-registry-v1.json", + "title": "Situated Experience Benchmark registry", + "type": "object", + "additionalProperties": false, + "required": ["schema_version", "benchmark_id", "name", "purpose", "families"], + "properties": { + "schema_version": {"const": "1.0.0"}, + "benchmark_id": {"const": "situated-experience-benchmark-v1"}, + "name": {"const": "Situated Experience Benchmark v1"}, + "purpose": {"type": "string", "minLength": 40}, + "families": { + "type": "array", + "minItems": 6, + "maxItems": 6, + "prefixItems": [ + {"$ref": "#/$defs/s1"}, + {"$ref": "#/$defs/s2"}, + {"$ref": "#/$defs/s3"}, + {"$ref": "#/$defs/s4"}, + {"$ref": "#/$defs/s5"}, + {"$ref": "#/$defs/s6"} + ], + "items": false + } + }, + "$defs": { + "family": { + "type": "object", + "additionalProperties": false, + "required": ["order", "id", "name", "status", "situated_knowledge", "screening_rules_path"], + "properties": { + "order": {"type": "integer", "minimum": 1, "maximum": 6}, + "id": {"type": "string"}, + "name": {"type": "string"}, + "status": {"enum": ["frozen_ready", "design_only"]}, + "situated_knowledge": {"type": "array", "minItems": 1, "items": {"type": "string"}}, + "screening_rules_path": {"type": "string"}, + "manifest_path": {"type": "string"} + } + }, + "s1": {"allOf": [{"$ref": "#/$defs/family"}, {"properties": {"order": {"const": 1}, "id": {"const": "S1"}, "status": {"const": "frozen_ready"}}, "required": ["manifest_path"]}]}, + "s2": {"allOf": [{"$ref": "#/$defs/family"}, {"properties": {"order": {"const": 2}, "id": {"const": "S2"}, "status": {"const": "design_only"}}}]}, + "s3": {"allOf": [{"$ref": "#/$defs/family"}, {"properties": {"order": {"const": 3}, "id": {"const": "S3"}, "status": {"const": "design_only"}}}]}, + "s4": {"allOf": [{"$ref": "#/$defs/family"}, {"properties": {"order": {"const": 4}, "id": {"const": "S4"}, "status": {"const": "design_only"}}}]}, + "s5": {"allOf": [{"$ref": "#/$defs/family"}, {"properties": {"order": {"const": 5}, "id": {"const": "S5"}, "status": {"const": "design_only"}}}]}, + "s6": {"allOf": [{"$ref": "#/$defs/family"}, {"properties": {"order": {"const": 6}, "id": {"const": "S6"}, "status": {"const": "design_only"}}}]} + } +} diff --git a/experiments/situated-experience-benchmark-v1/schemas/candidate-screening.schema.json b/experiments/situated-experience-benchmark-v1/schemas/candidate-screening.schema.json new file mode 100644 index 0000000..eb1e6ab --- /dev/null +++ b/experiments/situated-experience-benchmark-v1/schemas/candidate-screening.schema.json @@ -0,0 +1,45 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://aeg.dev/schemas/situated-candidate-screening-v1.json", + "type": "object", + "additionalProperties": false, + "required": ["schema_version", "benchmark_id", "screened_before_execution", "candidates"], + "properties": { + "schema_version": {"const": "1.0.0"}, + "benchmark_id": {"const": "situated-experience-benchmark-v1"}, + "screened_before_execution": {"const": true}, + "candidates": { + "type": "array", + "minItems": 2, + "items": { + "type": "object", + "additionalProperties": false, + "required": ["candidate_id", "family", "status", "title", "public_evidence", "decision_reasons"], + "properties": { + "candidate_id": {"type": "string", "minLength": 4}, + "family": {"enum": ["S1", "S2", "S3", "S4", "S5", "S6"]}, + "status": {"enum": ["accepted", "rejected"]}, + "title": {"type": "string", "minLength": 8}, + "public_evidence": {"type": "array", "minItems": 1, "items": {"type": "string"}}, + "decision_reasons": {"type": "array", "minItems": 1, "items": {"type": "string"}}, + "pair_id": {"type": "string"}, + "rejection_code": { + "enum": [ + "family_mismatch", + "trivial_static_migration", + "oracle_not_reproducible_offline", + "regression_coverage_incomplete", + "source_transfer_relation_missing", + "license_or_provenance_unresolved", + "fixture_too_large_or_unsafe" + ] + } + }, + "allOf": [ + {"if": {"properties": {"status": {"const": "accepted"}}}, "then": {"required": ["pair_id"], "not": {"required": ["rejection_code"]}}}, + {"if": {"properties": {"status": {"const": "rejected"}}}, "then": {"required": ["rejection_code"], "not": {"required": ["pair_id"]}}} + ] + } + } + } +} diff --git a/experiments/situated-experience-benchmark-v1/schemas/experience.schema.json b/experiments/situated-experience-benchmark-v1/schemas/experience.schema.json new file mode 100644 index 0000000..3e725a9 --- /dev/null +++ b/experiments/situated-experience-benchmark-v1/schemas/experience.schema.json @@ -0,0 +1,27 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://aeg.dev/schemas/situated-experience-v1.json", + "title": "Situated Experience Benchmark compact experience", + "type": "object", + "additionalProperties": false, + "required": [ + "context", + "version_constraints", + "failed_approach", + "invalidating_evidence", + "recovery_principle", + "validated_outcome", + "applicability_conditions", + "known_invalidation_conditions" + ], + "properties": { + "context": {"type": "string", "minLength": 20}, + "version_constraints": {"type": "string", "minLength": 20}, + "failed_approach": {"type": "string", "minLength": 20}, + "invalidating_evidence": {"type": "string", "minLength": 20}, + "recovery_principle": {"type": "string", "minLength": 20}, + "validated_outcome": {"type": "string", "minLength": 20}, + "applicability_conditions": {"type": "string", "minLength": 20}, + "known_invalidation_conditions": {"type": "string", "minLength": 20} + } +} diff --git a/experiments/situated-experience-benchmark-v1/schemas/manifest.schema.json b/experiments/situated-experience-benchmark-v1/schemas/manifest.schema.json new file mode 100644 index 0000000..639dd94 --- /dev/null +++ b/experiments/situated-experience-benchmark-v1/schemas/manifest.schema.json @@ -0,0 +1,55 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://aeg.dev/schemas/situated-s1-manifest-v1.json", + "type": "object", + "additionalProperties": false, + "required": ["schema_version", "benchmark_id", "name", "status", "family", "frozen_at", "protocol", "promotion_criteria", "stop_conditions", "pairs"], + "properties": { + "schema_version": {"const": "1.0.0"}, + "benchmark_id": {"const": "situated-experience-benchmark-v1"}, + "name": {"const": "Situated Experience Benchmark v1"}, + "status": {"const": "frozen-before-arm-execution"}, + "family": {"const": "S1"}, + "frozen_at": {"type": "string", "format": "date-time"}, + "protocol": { + "type": "object", + "additionalProperties": false, + "required": ["replicates_per_arm", "modes", "randomization_seed", "model", "task_input_policy", "budget", "evaluation_thresholds", "measurement_contract", "arm_orders"], + "properties": { + "replicates_per_arm": {"const": 3}, + "modes": {"const": ["control", "aeg-assisted"]}, + "randomization_seed": {"type": "string"}, + "model": {"type": "string"}, + "task_input_policy": {"type": "string"}, + "budget": {"type": "object", "additionalProperties": false, "required": ["wall_time_seconds", "max_completed_commands", "max_attempts"], "properties": {"wall_time_seconds": {"type": "integer", "minimum": 1}, "max_completed_commands": {"type": "integer", "minimum": 1}, "max_attempts": {"type": "integer", "minimum": 1}}}, + "evaluation_thresholds": {"type": "object", "additionalProperties": false, "required": ["effort_change_fraction", "max_treatment_token_overhead_fraction", "minimum_interpretable_repair_path_improvements", "negative_transfer_effort_measure_count"], "properties": {"effort_change_fraction": {"const": 0.2}, "max_treatment_token_overhead_fraction": {"const": 0.3}, "minimum_interpretable_repair_path_improvements": {"const": 1}, "negative_transfer_effort_measure_count": {"const": 2}}}, + "measurement_contract": {"type": "string"}, + "arm_orders": {"type": "object", "additionalProperties": {"type": "array", "minItems": 3, "maxItems": 3, "items": {"type": "array", "minItems": 2, "maxItems": 2, "items": {"enum": ["control", "aeg-assisted"]}}}} + } + }, + "promotion_criteria": {"type": "array", "minItems": 5, "items": {"type": "string"}}, + "stop_conditions": {"type": "array", "minItems": 6, "items": {"type": "string"}}, + "pairs": { + "type": "array", + "minItems": 2, + "maxItems": 2, + "items": { + "type": "object", + "additionalProperties": false, + "required": ["pair_id", "title", "project", "license", "fixture_kind", "source", "transfer", "experience_path", "experience_sha256", "source_fixture_sha256", "source_human_patch_sha256", "source_hidden_tests_sha256", "agent_fixture_sha256", "human_patch_sha256", "hidden_tests_sha256", "source_public_test_command", "public_test_command", "hidden_test_command", "source_initial_failure_pattern", "initial_failure_pattern", "historical_failure_paths", "environment_assumptions"], + "properties": { + "pair_id": {"type": "string"}, "title": {"type": "string"}, "project": {"type": "string"}, "license": {"type": "string"}, "fixture_kind": {"const": "dependency-free-public-extract"}, + "source": {"$ref": "#/$defs/public_task"}, "transfer": {"$ref": "#/$defs/public_task"}, + "experience_path": {"type": "string"}, "experience_sha256": {"$ref": "#/$defs/sha"}, "source_fixture_sha256": {"$ref": "#/$defs/sha"}, "source_human_patch_sha256": {"$ref": "#/$defs/sha"}, "source_hidden_tests_sha256": {"$ref": "#/$defs/sha"}, "agent_fixture_sha256": {"$ref": "#/$defs/sha"}, "human_patch_sha256": {"$ref": "#/$defs/sha"}, "hidden_tests_sha256": {"$ref": "#/$defs/sha"}, + "source_public_test_command": {"type": "string"}, "public_test_command": {"type": "string"}, "hidden_test_command": {"type": "string"}, "source_initial_failure_pattern": {"type": "string"}, "initial_failure_pattern": {"type": "string"}, + "historical_failure_paths": {"type": "array", "minItems": 1, "items": {"type": "object", "additionalProperties": false, "required": ["id", "description", "pattern"], "properties": {"id": {"type": "string"}, "description": {"type": "string"}, "pattern": {"type": "string"}}}}, + "environment_assumptions": {"type": "array", "minItems": 1, "items": {"type": "string"}} + } + } + } + }, + "$defs": { + "sha": {"type": "string", "pattern": "^[0-9a-f]{64}$"}, + "public_task": {"type": "object", "additionalProperties": false, "required": ["bug_id", "buggy_commit", "fixed_commit", "fixed_at", "evidence_url"], "properties": {"bug_id": {"type": "string"}, "buggy_commit": {"type": "string", "pattern": "^[0-9a-f]{40}$"}, "fixed_commit": {"type": "string", "pattern": "^[0-9a-f]{40}$"}, "fixed_at": {"type": "string", "format": "date-time"}, "evidence_url": {"type": "string", "format": "uri"}}} + } +} diff --git a/experiments/situated-experience-benchmark-v1/test_benchmark.py b/experiments/situated-experience-benchmark-v1/test_benchmark.py new file mode 100644 index 0000000..068365b --- /dev/null +++ b/experiments/situated-experience-benchmark-v1/test_benchmark.py @@ -0,0 +1,190 @@ +#!/usr/bin/env python3 +"""Adversarial tests for Situated Experience Benchmark v1.""" + +import importlib.util +import json +import os +import subprocess +import tempfile +import unittest +from pathlib import Path + + +HERE = Path(__file__).resolve().parent +SPEC = importlib.util.spec_from_file_location("situated_runner", HERE / "run_benchmark.py") +runner = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(runner) + + +class SituatedBenchmarkTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.manifest = runner.load_json(runner.MANIFEST) + + def package(self, root, pair_id="s1-01-scrapy-cookiejar", replicate=1, mode="control"): + pair = runner.find_pair(self.manifest, pair_id) + output = root / runner.arm_id(pair_id, replicate, mode) + runner.package_arm(self.manifest, pair, replicate, mode, output) + return pair, output + + def test_frozen_manifest_and_preflight_validate(self): + validated = runner.validate() + self.assertEqual(validated["accepted_pairs"], 2) + self.assertEqual(validated["planned_arms"], 12) + ready = runner.preflight() + self.assertEqual(ready["human_patches_verified"], 4) + self.assertEqual(ready["arm_bundles_audited"], 12) + + def test_source_transfer_pairs_are_natural_and_non_identical(self): + for pair in self.manifest["pairs"]: + self.assertLess(runner.parse_time(pair["source"]["fixed_at"]), runner.parse_time(pair["transfer"]["fixed_at"])) + self.assertNotEqual(pair["source"]["fixed_commit"], pair["transfer"]["fixed_commit"]) + self.assertNotEqual(pair["source_human_patch_sha256"], pair["human_patch_sha256"]) + + def test_control_and_assisted_bundle_separation(self): + with tempfile.TemporaryDirectory(prefix="seb-test-bundles-") as raw: + root = Path(raw) + pair, control = self.package(root, mode="control") + _, assisted = self.package(root, mode="aeg-assisted") + control_envelope = runner.load_json(control / "arm.json") + assisted_envelope = runner.load_json(assisted / "arm.json") + self.assertNotIn("experience", control_envelope) + self.assertEqual(set(assisted_envelope["experience"]), set(runner.EXPERIENCE_FIELDS)) + serialized = json.dumps(assisted_envelope) + self.assertNotIn(pair["transfer"]["fixed_commit"], serialized) + self.assertNotIn(pair["human_patch_sha256"], serialized) + + def test_evaluator_and_cross_arm_artifacts_fail_closed(self): + with tempfile.TemporaryDirectory(prefix="seb-test-adversarial-") as raw: + root = Path(raw) + pair, bundle = self.package(root) + hidden = bundle / "workspace" / "test_hidden.py" + hidden.write_text("raise AssertionError('leaked evaluator')\n", encoding="utf-8") + with self.assertRaises(runner.ProtocolError): + runner.audit_bundle(self.manifest, pair, bundle) + with tempfile.TemporaryDirectory(prefix="seb-test-cross-arm-") as raw: + root = Path(raw) + pair, bundle = self.package(root) + (bundle / "workspace" / "prior-arm.patch").write_text("sentinel\n", encoding="utf-8") + with self.assertRaises(runner.ProtocolError): + runner.audit_bundle(self.manifest, pair, bundle) + + def test_treatment_payload_cannot_add_patch_fields(self): + with tempfile.TemporaryDirectory(prefix="seb-test-experience-") as raw: + root = Path(raw) + pair, bundle = self.package(root, mode="aeg-assisted") + envelope_path = bundle / "arm.json" + envelope = runner.load_json(envelope_path) + envelope["experience"]["final_patch"] = "return the transfer fix" + runner.write_json(envelope_path, envelope) + with self.assertRaises(runner.ProtocolError): + runner.audit_bundle(self.manifest, pair, bundle) + + def test_worker_probe_rejects_credential_environment(self): + with tempfile.TemporaryDirectory(prefix="seb-test-worker-") as raw: + root = Path(raw) + _, bundle = self.package(root) + env = { + "PATH": os.environ.get("PATH", "/usr/bin:/bin"), + "HOME": str(root / "home"), + "TMPDIR": str(root / "tmp"), + "MODEL_API_KEY": "sentinel", + } + result = subprocess.run( + ["python3", str(bundle / "arm_worker.py"), "probe", "--bundle", str(bundle)], + env=env, + text=True, + capture_output=True, + check=False, + ) + self.assertNotEqual(result.returncode, 0) + self.assertIn("credential-shaped", result.stderr) + + def test_worker_probe_rejects_readable_sibling_arm(self): + with tempfile.TemporaryDirectory(prefix="seb-test-sibling-") as raw: + root = Path(raw) + _, bundle = self.package(root) + (root / "other-arm").mkdir() + env = { + "PATH": os.environ.get("PATH", "/usr/bin:/bin"), + "HOME": str(root / "home"), + "TMPDIR": str(root / "tmp"), + "SEB_RUNNER_ROOT": str(root), + } + result = subprocess.run( + ["python3", str(bundle / "arm_worker.py"), "probe", "--bundle", str(bundle)], + env=env, + text=True, + capture_output=True, + check=False, + ) + self.assertNotEqual(result.returncode, 0) + self.assertIn("other-arm", result.stderr) + + def test_mode_selector_rejects_unknown_mode(self): + result = subprocess.run( + ["python3", str(HERE / "run_benchmark.py"), "package-arm", "--pair", "s1-01-scrapy-cookiejar", "--replicate", "1", "--mode", "unknown", "--output", "/tmp/never-created-seb-mode"], + text=True, + capture_output=True, + check=False, + ) + self.assertNotEqual(result.returncode, 0) + self.assertIn("invalid choice", result.stderr) + + def test_hidden_evaluator_marks_human_transfer_patch_regression_free(self): + pair = runner.find_pair(self.manifest, "s1-02-fastapi-pydantic") + with tempfile.TemporaryDirectory(prefix="seb-test-evaluator-") as raw: + root = Path(raw) + arm_output = root / "arm-output" + arm_output.mkdir() + patch = runner.fixture_root(pair) / "transfer" / "evaluator" / "human.patch" + (arm_output / "patch.diff").write_bytes(patch.read_bytes()) + (arm_output / "events.jsonl").write_text("", encoding="utf-8") + digest = "0" * 64 + runner.write_json( + arm_output / "arm-result.json", + { + "schema_version": "1.0.0", + "benchmark_id": "situated-experience-benchmark-v1", + "family": "S1", + "pair_id": pair["pair_id"], + "replicate": 1, + "mode": "control", + "evaluation_status": "captured", + "input_hashes": {"manifest": digest, "agent_fixture": digest, "task_prompt": digest}, + "budget": self.manifest["protocol"]["budget"], + "regression_free_success": None, + "attempts": 1, + "completed_commands": 2, + "tests_run": [], + "files_inspected": ["form_extractor.py"], + "files_changed": ["form_extractor.py"], + "patch_size": {"added_lines": 4, "deleted_lines": 1, "files": 1}, + "wall_time_ms": 1, + "tokens": {"input": None, "output": None, "unavailable_reason": "fixture"}, + "failed_historical_paths_repeated": [], + "environment_assumptions_checked": [], + "experiences": [{"experience_id": None, "disposition": "abstained", "reason": "control"}], + "negative_transfer": None, + "evaluator_findings": ["hidden evaluation pending"], + "limitations": [], + }, + ) + evaluated = runner.evaluate_arm(pair, 1, "control", arm_output, root / "evaluated.json") + self.assertTrue(evaluated["regression_free_success"]) + self.assertEqual(evaluated["evaluation_status"], "evaluated") + + def test_schedule_contains_frozen_order_and_twelve_single_arm_bundles(self): + with tempfile.TemporaryDirectory(prefix="seb-test-schedule-") as raw: + output = Path(raw) / "schedule" + plan = runner.schedule_s1(output) + self.assertEqual(plan["arm_count"], 12) + expected = [] + for pair in self.manifest["pairs"]: + for replicate, order in enumerate(self.manifest["protocol"]["arm_orders"][pair["pair_id"]], 1): + expected.extend(runner.arm_id(pair["pair_id"], replicate, mode) for mode in order) + self.assertEqual([arm["arm_id"] for arm in plan["arms"]], expected) + + +if __name__ == "__main__": + unittest.main() diff --git a/experiments/v0.1.6-product-proof/README.md b/experiments/v0.1.6-product-proof/README.md new file mode 100644 index 0000000..f2f7a3f --- /dev/null +++ b/experiments/v0.1.6-product-proof/README.md @@ -0,0 +1,57 @@ +# AEG v0.1.6 product-proof preparation + +Status: **prepared only; not frozen; 0/3 arms executed**. + +This directory prepares one bounded three-arm test of the v0.1.6 product hypothesis: whether a top-1 verified capsule changes the outcome or effort on a naturally occurring public repair when compared with both no added context and fixed generic debugging advice. + +It does not authorize or run a model, spend money, select a target, inspect a human patch, or change the verified library. + +## Three arms + +1. **Baseline:** the frozen public task, code, failure, budget, and objective checks; no AEG records or added advice. +2. **Fixed generic:** identical inputs plus [`generic-advice.txt`](generic-advice.txt); no AEG record access. +3. **AEG top-1:** identical inputs plus only the automatically retrieved and frozen top-1 guarded capsule. + +Model, settings, prompt template, budget, environment, focused checks, regression checks, and evaluation are identical. The intentional context difference is the arm definition above. + +## Freeze gate before any execution + +A separate authorized phase must freeze all currently `null` fields in [`protocol.json`](protocol.json) before an arm can run: + +- a naturally occurring target from a predeclared candidate pool in a public repository with an explicit license; +- repository and buggy commit, task hash, objective check hashes, and freeze time before inspecting its human patch; +- a source experience whose timestamp predates the target; +- verified-library hash, fixed 0.0500 threshold, retrieval score, top-1 experience, and guarded-capsule hash; +- identical model, settings, prompt template, budget, oracle, and randomized arm order. + +Exclude eBay, private, internal, proprietary, credentialed, or capsule-derived targets. If no verified result clears the frozen threshold, record a retrieval abstention and stop without executing any arm. + +Each arm must use a separate sanitized repository and worktree directory with no shared Git metadata, writable cache, AEG access for controls, human patch, other-arm artifacts, or evaluator feedback. + +## Results + +[`result.schema.json`](result.schema.json) requires: + +- success, attempts, completed sanitized commands, and test executions; +- focused and regression check results; +- non-cached tokens and duration; +- repository-relative files inspected and changed; +- patch hash and identical-setting hashes; +- limitations, protocol deviations, and a privacy attestation. + +Publish negative, neutral, abstention, and protocol-deviation outcomes. Do not publish code, prompts, task text, raw logs, ratings, receipts, or private data. Do not change thresholds or outcome rules after observing a result. + +## Validation only + +The extension test suite validates both experiment schemas and checks valid completed, abstention, missing-measurement, and wrong-arm-order cases: + +```bash +cd integrations/vscode +npm test +``` + +This command does not execute an experiment arm. + +## Evidence boundary + +AEG currently contains two verified records in two narrow task families. The bundled challenge demonstrates the interaction flow, not a performance benefit. Prior controlled and transfer evidence was neutral or negative. Nothing in this preparation supports a claim of improved success, speed, cost, PMF, adoption, or generalization. diff --git a/experiments/v0.1.6-product-proof/STATUS.md b/experiments/v0.1.6-product-proof/STATUS.md new file mode 100644 index 0000000..611c2af --- /dev/null +++ b/experiments/v0.1.6-product-proof/STATUS.md @@ -0,0 +1,94 @@ +# AEG v0.1.6 product-proof status + +Last updated: 2026-08-14 (America/Los_Angeles). + +## Current phase + +The clean-profile founder usability/discoverability gate **passed on +2026-08-14** using VSIX SHA-256 +`18ef493b9290e28832e54527d7fb92624387a17d749ec228b60087c3b6917224`. +PR #28 remains Draft. The product-proof experiment remains **prepared, not +frozen; 0/3 arms executed**. + +## Founder acceptance record — PASS + +- **Date:** 2026-08-14 (America/Los_Angeles). +- **Artifact:** VSIX SHA-256 + `18ef493b9290e28832e54527d7fb92624387a17d749ec228b60087c3b6917224`. +- **PASS:** the first empty workspace automatically opened **AEG + verified-experience proof loop**, and the founder completed all five steps + through local feedback creation. +- **PASS:** the local feedback file was created successfully. +- **PASS:** a second workspace using the same profile did not force the + walkthrough open. +- **PASS:** **AEG: Open Founder Proof Walkthrough** manually reopened it. +- **Boundary:** no repair-performance arm ran; this validates usability and + discoverability only. + +## Completed work + +- Audited the v0.1.5 workflow, required evidence, both dogfood decisions, the verified challenge, natural-transfer benchmark, and GitHub Issue #12. +- Preserved the two-record verified library and fixed 0.0500 retrieval threshold. +- Added one dominant verified-experience command, honest coverage, explicit abstention, guarded handoff instructions, enforced validation-before-rating, and local query/experience/outcome-linked feedback. +- Moved Playwright, Repair Lab, skill tools, the synthetic challenge, and legacy commands under **Advanced** without deleting command IDs. +- Added a five-step founder walkthrough using supported VS Code contribution points. +- Diagnosed the closed-window command-line VSIX install gap: VS Code's generic + auto-open path only sees extensions installed into the focused workbench + session, while AEG previously had no startup activation or owned first-run + state. +- Added deferred startup activation, a versioned global marker with recoverable + `opening`/`opened`/`failed` states, manual reopen, and a visible **AEG: Start + here** status-bar fallback. +- Prepared the non-executing baseline/fixed-generic/AEG-top-1 protocol, generic advice, strict schemas, UX audit, and founder acceptance gate. + +## Tests + +- Untouched v0.1.5 baseline: 20/20 extension tests passed; TypeScript compiled; baseline VSIX packaged. +- v0.1.6 extension, first-run/UX-transition, and schema suite: 39/39 tests passed; TypeScript compiled. +- Retrieval and verified-experience validation: 11/11 Python tests passed; two records validated with two unique IDs. +- Site regression suite: 8/8 tests passed. +- Disposable-profile VS Code 1.133.0 install-path smoke: the local VSIX + installed as `agentexperiencegraph.agent-experience-graph@0.1.6`; the first + workspace activated AEG through `onStartupFinished`, persisted the versioned + `opened` marker, and selected the AEG founder walkthrough; a second workspace + with the same profile activated AEG without selecting that walkthrough and + left the marker unchanged. +- Patch whitespace and private-path/credential scans: passed. +- `experiences/verified.json` diff against `origin/main`: empty. +- VSIX content inspection: version 0.1.6, deferred startup activation, + compiled first-run state machine, all existing command IDs, five walkthrough + files, and exactly two verified records; packaged record hash matches the + source library. + +## VSIX + +- Path: `integrations/vscode/agent-experience-graph-0.1.6.vsix` (local ignored build artifact) +- SHA-256: `18ef493b9290e28832e54527d7fb92624387a17d749ec228b60087c3b6917224` + +## Blockers + +- No founder usability/discoverability blocker remains open. +- Product-proof execution is intentionally not authorized in this checkpoint; + the protocol remains prepared, not frozen, with 0/3 arms executed. + +## Publication handoff + +- Branch: `codex/v0.1.6-founder-ready-proof-loop` +- Commit chain through the onboarding fix: `f8835c0`, `6e13568`, and `5c0b010`. +- Draft PR: +- Release/merge/Marketplace publication: not performed + +## Decisions + +- Use clipboard plus explicit Chat paste-and-run instructions. The documented editor-chat command is not a reliable normal-Chat handoff, and no private workbench command is used. +- Treat abstention as a successful retrieval decision and inject no generic fallback. +- Require a recorded validation outcome before usefulness feedback. +- Keep all feedback and receipts local; upload nothing from the extension. +- Do not select a target or execute an arm in v0.1.6. + +## Next authorized decision + +The founder UX checkpoint is complete. Selecting a target, freezing the +product-proof protocol, or executing any arm requires separate explicit +authorization. Do not run D001 or another arm as part of this acceptance +record. diff --git a/experiments/v0.1.6-product-proof/UX-ACCEPTANCE.md b/experiments/v0.1.6-product-proof/UX-ACCEPTANCE.md new file mode 100644 index 0000000..7e65db2 --- /dev/null +++ b/experiments/v0.1.6-product-proof/UX-ACCEPTANCE.md @@ -0,0 +1,107 @@ +# v0.1.6 founder UX acceptance + +This is a usability gate, not an experiment arm and not evidence of repair +benefit. + +## Acceptance record — PASS + +Founder test date: **2026-08-14** (America/Los_Angeles). + +Tested VSIX SHA-256: +`18ef493b9290e28832e54527d7fb92624387a17d749ec228b60087c3b6917224`. + +- **PASS — clean profile.** The founder tested with a fresh VS Code profile. +- **PASS — first-run discovery.** The first empty workspace automatically + opened **AEG verified-experience proof loop** without manual command + discovery. +- **PASS — complete proof loop.** The founder completed all five steps: task + entry → verified record inspection → capsule copy → validation outcome → + local feedback. +- **PASS — local persistence.** The local feedback file was created + successfully. +- **PASS — non-repeating behavior.** A second workspace using the same profile + did not force the walkthrough open. +- **PASS — manual reopen.** **AEG: Open Founder Proof Walkthrough** reopened the + walkthrough successfully. +- **PASS — experiment boundary.** No repair-performance experiment arm was + run. + +**Founder usability/discoverability gate: PASSED.** The product-proof +experiment remains **prepared, not frozen; 0/3 arms executed**. + +This result validates onboarding usability and discoverability only. It is not +evidence of better repair success, speed, cost, adoption, product-market fit, +or generalization. + +## Clean-profile test precondition + +An uninstall is not sufficient because extension global state can survive in a +VS Code profile. Use a disposable profile and extensions directory that have +never contained AEG: + +```bash +code \ + --user-data-dir /absolute/path/to/aeg-founder-retest/user-data \ + --extensions-dir /absolute/path/to/aeg-founder-retest/extensions \ + --install-extension /absolute/path/to/agent-experience-graph-0.1.6.vsix \ + --force +``` + +Keep those two directories for both-window checks. Use a fresh disposable local +workspace with at least one folder, and do not give the founder a README, +command name, Command Palette instruction, or outside navigation hint. + +## Founder test procedure + +1. Launch the fresh workspace with the same disposable `--user-data-dir` and + `--extensions-dir`. Start a three-minute timer when the window is usable. +2. Confirm the AEG founder walkthrough opens without any manual command. Also + confirm the status bar exposes **AEG: Start here**, the sidebar reports + **2 records · 2 task families**, and legacy tools remain collapsed under + **Advanced**. +3. Start the primary path and enter: + `Keepalive control fails after active stream ownership moved behind a protocol object; repair the public wrapper so it delegates through the protocol without using its stale socket field.` +4. Select the above-threshold result. Confirm the panel keeps the original + query visible and shows matching phrases, score, verified outcome, public + provenance, constraints, limitations, and the “guidance, not a guaranteed + answer” guardrail. +5. Select **Copy capsule**. Confirm the clipboard contains the capsule and the + panel says exactly how to open VS Code Chat, where to paste it, and that AEG + did not send or run it. +6. For this usability test, select **Did not apply**, then **Irrelevant**. + Confirm `.aeg/verified-experience-feedback.json` links the original query + summary, selected experience ID/task, validation outcome, rating, retrieval + score, and local-only flag. +7. Start again with: + `Change the website navigation background from white to blue and increase the logo size.` + Confirm **No relevant verified experience** is presented as a correct outcome + with score/threshold reasoning, coverage, and no injected fallback. +8. Close the first window. Launch a second normal workspace window with the same + disposable profile and extensions directory. Confirm the walkthrough does + not force itself open again and **AEG: Start here** remains visible. +9. Open the AEG Activity Bar view and select **Guided walkthrough**. Confirm the + walkthrough reopens manually without using the Command Palette. + +## Pass criteria + +- The first normal-workspace activation presents the walkthrough without + README, Command Palette knowledge, or outside instructions. +- The primary action remains visually discoverable as **AEG: Start here** if + automatic opening is skipped or fails. +- A second window in the same profile does not force the walkthrough open. +- The founder can reopen it from the AEG sidebar. +- Every matched step is ordered and understandable without guessing the next + action. +- The user cannot validate before copying or rate before validation. +- Match evidence and evidence limitations remain visible before handoff. +- Handoff uses only clipboard plus explicit instructions; no undocumented chat + command is invoked. +- Feedback connects query, selected experience, observed validation outcome, + and rating in a local workspace file. +- Abstention is calm, explicit, and informative. +- No network request, upload, model run, Repair Lab run, experiment arm, + release, or Marketplace publish occurs. + +Any future regression on these items reopens the founder usability gate. Do not +reinterpret this usability pass as performance, adoption, product-market fit, +or generalization evidence. diff --git a/experiments/v0.1.6-product-proof/UX-AUDIT.md b/experiments/v0.1.6-product-proof/UX-AUDIT.md new file mode 100644 index 0000000..c9d86d3 --- /dev/null +++ b/experiments/v0.1.6-product-proof/UX-AUDIT.md @@ -0,0 +1,80 @@ +# v0.1.5 audit and v0.1.6 founder-discovery incident + +Audit date: 2026-08-11. Baseline: `origin/main` at `544874c`, extension version 0.1.5. + +The untouched baseline compiled, passed all 20 extension tests, and produced a local VSIX. The individual features worked, but a first-time user had no clearly dominant workflow: + +- **Try a Verified Experience**, **Open Verified Experience Challenge**, Playwright diagnosis, outcome marking, receipt history, Repair Lab, and skill tools appeared at similar prominence across the sidebar, command palette, editor context menu, view title, and status bar. +- The Activity Bar view was named **Verified Experience & Playwright**, so product identity and a legacy diagnosis vertical competed before the user entered a task. +- The primary verified card explained a match and copied a guarded capsule, but did not tell the user exactly which chat input to use or advance through an explicit validation state. +- Four usefulness buttons were available before any observed outcome was recorded. Feedback linked the query and experience, but not a validation result. +- The UI did not foreground the two-record/two-family coverage boundary. Abstention appeared as a transient notification, which could read like failure rather than calibrated retrieval. +- The README carried the only end-to-end explanation; there was no first-install walkthrough that tracked completion. + +The v0.1.6 decision is one reversible path: **Start with Verified Experience → inspect match or abstention → copy with explicit paste instructions → record objective validation → save local rating**. Playwright, Repair Lab, skill discovery, and all legacy command IDs remain available under **Advanced**. + +The supported VS Code command list documents `vscode.editorChat.start`, which starts editor chat, but does not document a stable command for an extension to open and prefill the normal Chat view. v0.1.6 therefore uses the supported clipboard API and tells the user exactly how to open Chat, paste, and run; it does not depend on a private workbench command. + +## v0.1.6 founder result: discovery gate failed + +The founder installed the local v0.1.6 VSIX successfully. After receiving +external Command Palette instructions, the founder completed the verified path +end to end: match, evidence inspection, guarded capsule copy, validation, and a +local feedback record. The saved row retained the query, experience ID, score, +`validationOutcome: "not-applied"`, `rating: "Irrelevant"`, and +`localOnly: true`. + +That functional result does not pass the usability gate. The first-install +walkthrough did not appear, and no primary AEG action drew the founder into the +flow. The overall founder gate is **failed; re-test required**. + +## Precise root cause + +The v0.1.6 package contributed a valid walkthrough, but relied entirely on VS +Code's generic “open on install” behavior. In VS Code, that behavior is driven +by an in-memory set populated by the focused window's extension-install event. +Only a newly registered walkthrough whose extension ID is in that same-session +set is selected for automatic opening. A command-line VSIX install performed +while all VS Code windows are closed has no focused workbench session to receive +the install event. On the next launch, the walkthrough is registered as new, +but its extension ID is not in the session-install set, so it is not opened. + +AEG had no startup activation event and no first-run code of its own. Its only +activation paths were the AEG view and AEG commands—the exact surfaces the +founder did not yet know to use. Therefore neither the walkthrough nor the +status-bar action was guaranteed to appear in the real install-then-launch path. + +## Minimal first-run fix + +- `onStartupFinished` activates AEG after startup without blocking the startup + path. +- First activation in a normal workspace opens the founder walkthrough through + a fire-and-forget call. +- A global, versioned marker records `opening`, `opened`, or `failed`. `opened` + suppresses later windows; `failed` retries; a one-minute stale `opening` + marker recovers from a killed or reloaded extension host without allowing two + simultaneous windows to race the walkthrough open. +- Marker reads and writes are best effort. Any failure leaves startup running + and the visible fallback available. +- **AEG: Start here** stays in the status bar after deferred activation and + invokes the primary verified-experience path. The AEG sidebar retains a + direct **Guided walkthrough** item and the manual reopen command remains + registered. + +## Follow-up founder acceptance — PASS + +On 2026-08-14, the founder re-tested VSIX SHA-256 +`18ef493b9290e28832e54527d7fb92624387a17d749ec228b60087c3b6917224` +in a clean VS Code profile: + +- the first empty workspace automatically opened **AEG verified-experience + proof loop**; +- all five steps completed and the local feedback file was created; +- a second workspace using the same profile did not force the walkthrough + open; and +- **AEG: Open Founder Proof Walkthrough** manually reopened it successfully. + +The founder usability/discoverability gate is therefore **passed**. No +repair-performance arm ran. The product-proof experiment remains prepared, not +frozen, with 0/3 arms, and this usability result supports no claim about repair +success, speed, cost, adoption, product-market fit, or generalization. diff --git a/experiments/v0.1.6-product-proof/generic-advice.txt b/experiments/v0.1.6-product-proof/generic-advice.txt new file mode 100644 index 0000000..b9965f4 --- /dev/null +++ b/experiments/v0.1.6-product-proof/generic-advice.txt @@ -0,0 +1 @@ +Reproduce the failure before editing. Read the failing test, the nearby production code, and relevant history available in the frozen target. Form a small set of hypotheses and test the highest-signal one first. Make the smallest maintainable change that addresses the observed cause. Run the focused test after each repair attempt, then run the frozen regression checks. Do not modify tests, weaken assertions, inspect a human fix, or claim success without the objective checks. diff --git a/experiments/v0.1.6-product-proof/protocol.json b/experiments/v0.1.6-product-proof/protocol.json new file mode 100644 index 0000000..a2a7e33 --- /dev/null +++ b/experiments/v0.1.6-product-proof/protocol.json @@ -0,0 +1,142 @@ +{ + "$schema": "./protocol.schema.json", + "schemaVersion": "1.0.0", + "protocolId": "aeg-v0.1.6-product-proof", + "status": "prepared-not-frozen", + "executionAuthorized": false, + "evidenceBoundary": { + "verifiedRecordCount": 2, + "verifiedLibraryMutable": false, + "challengeEvidence": "interaction-only", + "priorTransferEvidence": "neutral-or-negative", + "claimsExcluded": [ + "improved-success", + "improved-speed", + "improved-cost", + "product-market-fit", + "adoption", + "generalization" + ] + }, + "arms": [ + { + "id": "baseline", + "label": "Baseline", + "additionalContext": null, + "aegLibraryAccess": false, + "targetInputs": "identical-frozen", + "objectiveOracle": "identical-frozen" + }, + { + "id": "fixed-generic", + "label": "Fixed generic debugging advice", + "additionalContext": "generic-advice.txt", + "aegLibraryAccess": false, + "targetInputs": "identical-frozen", + "objectiveOracle": "identical-frozen" + }, + { + "id": "aeg-top-1", + "label": "AEG top-1 verified capsule", + "additionalContext": "frozen-top-1-guarded-capsule", + "aegLibraryAccess": "top-1-only", + "targetInputs": "identical-frozen", + "objectiveOracle": "identical-frozen" + } + ], + "eligibility": { + "publicRepository": true, + "explicitLicense": true, + "excludedSources": [ + "ebay", + "private", + "internal", + "proprietary" + ], + "sourcePredatesTarget": true, + "targetFrozenBeforeHumanPatch": true, + "syntheticTargetForbidden": true, + "candidatePoolFrozenBeforeRetrieval": true + }, + "freeze": { + "requiredBeforeExecution": true, + "target": { + "repository": null, + "license": null, + "buggyCommitSha": null, + "taskHash": null, + "focusedTestHash": null, + "regressionTestHash": null, + "frozenAt": null + }, + "sourceExperience": { + "id": null, + "recordedAt": null, + "librarySha256": null, + "retrievalScore": null, + "retrievalThreshold": 0.05, + "capsuleSha256": null + }, + "controls": { + "model": null, + "settingsSha256": null, + "promptTemplateSha256": null, + "budgetSha256": null, + "oracleSha256": null, + "genericAdviceSha256": "8d90bc397194fa3a6c146914e36860ffe887a77fb8de63ed9e5212099fdbd022" + } + }, + "execution": { + "isolation": "separate sanitized repositories and worktree directories with no shared Git metadata or writable caches", + "freshContextPerArm": true, + "sharedCaches": false, + "armOrder": null, + "maxAttemptsPerArm": 3, + "maxDurationSecondsPerArm": 1500, + "humanPatchAvailableDuringArms": false, + "executeInThisRelease": false + }, + "measurements": [ + "success", + "attempts", + "commands", + "tests", + "nonCachedTokens", + "durationMs", + "filesInspected", + "filesChanged", + "patchHash" + ], + "evaluation": { + "focusedTestsRequired": true, + "regressionTestsRequired": true, + "sameOracleAcrossArms": true, + "thresholdChangesAfterResults": false, + "blindArmEvaluation": true, + "successDefinition": "The unchanged focused and regression checks pass without test edits or a protocol violation." + }, + "publication": { + "publishNegativeOutcomes": true, + "publishAbstentions": true, + "publishProtocolDeviations": true, + "forbiddenArtifacts": [ + "code", + "prompts", + "task-text", + "raw-logs", + "ratings", + "receipts", + "private-data" + ] + }, + "stopConditions": [ + "No verified result clears the frozen threshold.", + "The target is synthetic or constructed from a capsule.", + "The source experience does not predate the frozen target.", + "A public explicit license cannot be confirmed.", + "The human patch or another arm becomes visible.", + "Identical settings, budget, or objective oracles cannot be maintained.", + "Isolation, credentials, privacy, or authorization boundaries fail.", + "An experiment arm is requested before a separate execution authorization." + ] +} diff --git a/experiments/v0.1.6-product-proof/protocol.schema.json b/experiments/v0.1.6-product-proof/protocol.schema.json new file mode 100644 index 0000000..9dbf63d --- /dev/null +++ b/experiments/v0.1.6-product-proof/protocol.schema.json @@ -0,0 +1,216 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://aeg.local/experiments/v0.1.6-product-proof/protocol.schema.json", + "title": "AEG v0.1.6 product-proof protocol", + "type": "object", + "additionalProperties": false, + "required": [ + "schemaVersion", + "protocolId", + "status", + "executionAuthorized", + "evidenceBoundary", + "arms", + "eligibility", + "freeze", + "execution", + "measurements", + "evaluation", + "publication", + "stopConditions" + ], + "properties": { + "$schema": {"type": "string"}, + "schemaVersion": {"const": "1.0.0"}, + "protocolId": {"const": "aeg-v0.1.6-product-proof"}, + "status": {"const": "prepared-not-frozen"}, + "executionAuthorized": {"const": false}, + "evidenceBoundary": { + "type": "object", + "additionalProperties": false, + "required": ["verifiedRecordCount", "verifiedLibraryMutable", "challengeEvidence", "priorTransferEvidence", "claimsExcluded"], + "properties": { + "verifiedRecordCount": {"const": 2}, + "verifiedLibraryMutable": {"const": false}, + "challengeEvidence": {"const": "interaction-only"}, + "priorTransferEvidence": {"enum": ["neutral-or-negative"]}, + "claimsExcluded": { + "type": "array", + "minItems": 6, + "uniqueItems": true, + "items": {"enum": ["improved-success", "improved-speed", "improved-cost", "product-market-fit", "adoption", "generalization"]} + } + } + }, + "arms": { + "type": "array", + "minItems": 3, + "maxItems": 3, + "prefixItems": [ + {"$ref": "#/$defs/baselineArm"}, + {"$ref": "#/$defs/genericArm"}, + {"$ref": "#/$defs/aegArm"} + ], + "items": false + }, + "eligibility": { + "type": "object", + "additionalProperties": false, + "required": ["publicRepository", "explicitLicense", "excludedSources", "sourcePredatesTarget", "targetFrozenBeforeHumanPatch", "syntheticTargetForbidden", "candidatePoolFrozenBeforeRetrieval"], + "properties": { + "publicRepository": {"const": true}, + "explicitLicense": {"const": true}, + "excludedSources": { + "type": "array", + "minItems": 4, + "uniqueItems": true, + "items": {"enum": ["ebay", "private", "internal", "proprietary"]} + }, + "sourcePredatesTarget": {"const": true}, + "targetFrozenBeforeHumanPatch": {"const": true}, + "syntheticTargetForbidden": {"const": true}, + "candidatePoolFrozenBeforeRetrieval": {"const": true} + } + }, + "freeze": { + "type": "object", + "additionalProperties": false, + "required": ["requiredBeforeExecution", "target", "sourceExperience", "controls"], + "properties": { + "requiredBeforeExecution": {"const": true}, + "target": {"$ref": "#/$defs/unfrozenTarget"}, + "sourceExperience": {"$ref": "#/$defs/unfrozenSourceExperience"}, + "controls": {"$ref": "#/$defs/unfrozenControls"} + } + }, + "execution": { + "type": "object", + "additionalProperties": false, + "required": ["isolation", "freshContextPerArm", "sharedCaches", "armOrder", "maxAttemptsPerArm", "maxDurationSecondsPerArm", "humanPatchAvailableDuringArms", "executeInThisRelease"], + "properties": { + "isolation": {"const": "separate sanitized repositories and worktree directories with no shared Git metadata or writable caches"}, + "freshContextPerArm": {"const": true}, + "sharedCaches": {"const": false}, + "armOrder": {"type": "null"}, + "maxAttemptsPerArm": {"type": "integer", "minimum": 1}, + "maxDurationSecondsPerArm": {"type": "integer", "minimum": 1}, + "humanPatchAvailableDuringArms": {"const": false}, + "executeInThisRelease": {"const": false} + } + }, + "measurements": { + "type": "array", + "uniqueItems": true, + "minItems": 9, + "items": {"enum": ["success", "attempts", "commands", "tests", "nonCachedTokens", "durationMs", "filesInspected", "filesChanged", "patchHash"]} + }, + "evaluation": { + "type": "object", + "additionalProperties": false, + "required": ["focusedTestsRequired", "regressionTestsRequired", "sameOracleAcrossArms", "thresholdChangesAfterResults", "blindArmEvaluation", "successDefinition"], + "properties": { + "focusedTestsRequired": {"const": true}, + "regressionTestsRequired": {"const": true}, + "sameOracleAcrossArms": {"const": true}, + "thresholdChangesAfterResults": {"const": false}, + "blindArmEvaluation": {"const": true}, + "successDefinition": {"type": "string", "minLength": 20} + } + }, + "publication": { + "type": "object", + "additionalProperties": false, + "required": ["publishNegativeOutcomes", "publishAbstentions", "publishProtocolDeviations", "forbiddenArtifacts"], + "properties": { + "publishNegativeOutcomes": {"const": true}, + "publishAbstentions": {"const": true}, + "publishProtocolDeviations": {"const": true}, + "forbiddenArtifacts": { + "type": "array", + "minItems": 7, + "uniqueItems": true, + "items": {"enum": ["code", "prompts", "task-text", "raw-logs", "ratings", "receipts", "private-data"]} + } + } + }, + "stopConditions": { + "type": "array", + "minItems": 7, + "uniqueItems": true, + "items": {"type": "string", "minLength": 8} + } + }, + "$defs": { + "armBase": { + "type": "object", + "additionalProperties": false, + "required": ["id", "label", "additionalContext", "aegLibraryAccess", "targetInputs", "objectiveOracle"], + "properties": { + "id": {"type": "string"}, + "label": {"type": "string", "minLength": 3}, + "additionalContext": {"type": ["string", "null"]}, + "aegLibraryAccess": {"type": ["boolean", "string"]}, + "targetInputs": {"const": "identical-frozen"}, + "objectiveOracle": {"const": "identical-frozen"} + } + }, + "baselineArm": { + "allOf": [ + {"$ref": "#/$defs/armBase"}, + {"properties": {"id": {"const": "baseline"}, "additionalContext": {"type": "null"}, "aegLibraryAccess": {"const": false}}} + ] + }, + "genericArm": { + "allOf": [ + {"$ref": "#/$defs/armBase"}, + {"properties": {"id": {"const": "fixed-generic"}, "additionalContext": {"const": "generic-advice.txt"}, "aegLibraryAccess": {"const": false}}} + ] + }, + "aegArm": { + "allOf": [ + {"$ref": "#/$defs/armBase"}, + {"properties": {"id": {"const": "aeg-top-1"}, "additionalContext": {"const": "frozen-top-1-guarded-capsule"}, "aegLibraryAccess": {"const": "top-1-only"}}} + ] + }, + "unfrozenTarget": { + "type": "object", + "additionalProperties": false, + "required": ["repository", "license", "buggyCommitSha", "taskHash", "focusedTestHash", "regressionTestHash", "frozenAt"], + "properties": { + "repository": {"type": "null"}, + "license": {"type": "null"}, + "buggyCommitSha": {"type": "null"}, + "taskHash": {"type": "null"}, + "focusedTestHash": {"type": "null"}, + "regressionTestHash": {"type": "null"}, + "frozenAt": {"type": "null"} + } + }, + "unfrozenSourceExperience": { + "type": "object", + "additionalProperties": false, + "required": ["id", "recordedAt", "librarySha256", "retrievalScore", "retrievalThreshold", "capsuleSha256"], + "properties": { + "id": {"type": "null"}, + "recordedAt": {"type": "null"}, + "librarySha256": {"type": "null"}, + "retrievalScore": {"type": "null"}, + "retrievalThreshold": {"const": 0.05}, + "capsuleSha256": {"type": "null"} + } + }, + "unfrozenControls": { + "type": "object", + "additionalProperties": false, + "required": ["model", "settingsSha256", "promptTemplateSha256", "budgetSha256", "oracleSha256", "genericAdviceSha256"], + "properties": { + "model": {"type": "null"}, + "settingsSha256": {"type": "null"}, + "promptTemplateSha256": {"type": "null"}, + "budgetSha256": {"type": "null"}, + "oracleSha256": {"type": "null"}, + "genericAdviceSha256": {"const": "8d90bc397194fa3a6c146914e36860ffe887a77fb8de63ed9e5212099fdbd022"} + } + } + } +} diff --git a/experiments/v0.1.6-product-proof/result.schema.json b/experiments/v0.1.6-product-proof/result.schema.json new file mode 100644 index 0000000..b453184 --- /dev/null +++ b/experiments/v0.1.6-product-proof/result.schema.json @@ -0,0 +1,188 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://aeg.local/experiments/v0.1.6-product-proof/result.schema.json", + "title": "AEG v0.1.6 product-proof triad result", + "type": "object", + "additionalProperties": false, + "required": [ + "schemaVersion", + "protocolId", + "taskId", + "recordedAt", + "outcome", + "sourceExperience", + "retrieval", + "armResults", + "decision", + "limitations", + "privacy" + ], + "properties": { + "$schema": {"type": "string"}, + "schemaVersion": {"const": "1.0.0"}, + "protocolId": {"const": "aeg-v0.1.6-product-proof"}, + "taskId": {"type": "string", "pattern": "^[a-z0-9][a-z0-9.-]+$"}, + "recordedAt": {"type": "string", "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T"}, + "outcome": {"enum": ["completed", "retrieval-abstention", "protocol-deviation"]}, + "sourceExperience": { + "oneOf": [ + {"type": "null"}, + { + "type": "object", + "additionalProperties": false, + "required": ["id", "recordedAt", "predatesTarget", "librarySha256", "capsuleSha256"], + "properties": { + "id": {"type": "string", "pattern": "^trace-"}, + "recordedAt": {"type": "string"}, + "predatesTarget": {"const": true}, + "librarySha256": {"$ref": "#/$defs/sha256"}, + "capsuleSha256": {"$ref": "#/$defs/sha256"} + } + } + ] + }, + "retrieval": { + "type": "object", + "additionalProperties": false, + "required": ["decision", "score", "threshold", "matchedFields"], + "properties": { + "decision": {"enum": ["top-1", "abstain"]}, + "score": {"type": ["number", "null"], "minimum": 0}, + "threshold": {"const": 0.05}, + "matchedFields": { + "type": "array", + "uniqueItems": true, + "items": {"enum": ["task", "reuse.retrievalTags", "reuse.recommendedFor", "lessons", "subtasks.description", "subtasks.lessons"]} + } + } + }, + "armResults": { + "type": "array", + "items": {"$ref": "#/$defs/armResult"} + }, + "decision": {"enum": ["supported", "neutral", "negative", "abstention", "protocol-deviation"]}, + "limitations": { + "type": "array", + "minItems": 1, + "items": {"type": "string", "minLength": 10} + }, + "privacy": { + "type": "object", + "additionalProperties": false, + "required": ["containsCode", "containsPrompts", "containsTaskText", "containsRawLogs", "containsRatings", "containsReceipts", "containsPrivateData"], + "properties": { + "containsCode": {"const": false}, + "containsPrompts": {"const": false}, + "containsTaskText": {"const": false}, + "containsRawLogs": {"const": false}, + "containsRatings": {"const": false}, + "containsReceipts": {"const": false}, + "containsPrivateData": {"const": false} + } + } + }, + "allOf": [ + { + "if": {"properties": {"outcome": {"const": "completed"}}}, + "then": { + "properties": { + "sourceExperience": {"type": "object"}, + "retrieval": {"properties": {"decision": {"const": "top-1"}, "score": {"type": "number", "minimum": 0.05}}}, + "armResults": { + "minItems": 3, + "maxItems": 3, + "prefixItems": [ + {"allOf": [{"$ref": "#/$defs/armResult"}, {"properties": {"arm": {"const": "baseline"}}}]}, + {"allOf": [{"$ref": "#/$defs/armResult"}, {"properties": {"arm": {"const": "fixed-generic"}}}]}, + {"allOf": [{"$ref": "#/$defs/armResult"}, {"properties": {"arm": {"const": "aeg-top-1"}}}]} + ], + "items": false + }, + "decision": {"enum": ["supported", "neutral", "negative"]} + } + } + }, + { + "if": {"properties": {"outcome": {"const": "retrieval-abstention"}}}, + "then": { + "properties": { + "sourceExperience": {"type": "null"}, + "retrieval": {"properties": {"decision": {"const": "abstain"}, "score": {"type": ["number", "null"], "exclusiveMaximum": 0.05}}}, + "armResults": {"maxItems": 0}, + "decision": {"const": "abstention"} + } + } + }, + { + "if": {"properties": {"outcome": {"const": "protocol-deviation"}}}, + "then": {"properties": {"decision": {"const": "protocol-deviation"}}} + } + ], + "$defs": { + "sha256": {"type": "string", "pattern": "^[a-f0-9]{64}$"}, + "repoPath": {"type": "string", "pattern": "^(?!/)(?!.*(?:^|/)\\.\\.(?:/|$)).+$"}, + "checkResult": { + "type": "object", + "additionalProperties": false, + "required": ["commandSha256", "status"], + "properties": { + "commandSha256": {"$ref": "#/$defs/sha256"}, + "status": {"enum": ["passed", "failed", "not-run"]} + } + }, + "armResult": { + "type": "object", + "additionalProperties": false, + "required": ["arm", "success", "attempts", "commands", "tests", "nonCachedTokens", "durationMs", "files", "patchHash", "settings", "protocolDeviation"], + "properties": { + "arm": {"enum": ["baseline", "fixed-generic", "aeg-top-1"]}, + "success": {"type": "boolean"}, + "attempts": {"type": "integer", "minimum": 0}, + "commands": { + "type": "object", + "additionalProperties": false, + "required": ["completed", "sanitized"], + "properties": { + "completed": {"type": "integer", "minimum": 0}, + "sanitized": {"type": "array", "items": {"type": "string", "minLength": 1}} + } + }, + "tests": { + "type": "object", + "additionalProperties": false, + "required": ["executions", "focused", "regression"], + "properties": { + "executions": {"type": "integer", "minimum": 0}, + "focused": {"$ref": "#/$defs/checkResult"}, + "regression": {"$ref": "#/$defs/checkResult"} + } + }, + "nonCachedTokens": {"type": "integer", "minimum": 0}, + "durationMs": {"type": "integer", "minimum": 0}, + "files": { + "type": "object", + "additionalProperties": false, + "required": ["inspected", "changed"], + "properties": { + "inspected": {"type": "array", "uniqueItems": true, "items": {"$ref": "#/$defs/repoPath"}}, + "changed": {"type": "array", "uniqueItems": true, "items": {"$ref": "#/$defs/repoPath"}} + } + }, + "patchHash": {"$ref": "#/$defs/sha256"}, + "settings": { + "type": "object", + "additionalProperties": false, + "required": ["model", "settingsSha256", "promptTemplateSha256", "budgetSha256", "oracleSha256"], + "properties": { + "model": {"type": "string", "minLength": 1}, + "settingsSha256": {"$ref": "#/$defs/sha256"}, + "promptTemplateSha256": {"$ref": "#/$defs/sha256"}, + "budgetSha256": {"$ref": "#/$defs/sha256"}, + "oracleSha256": {"$ref": "#/$defs/sha256"} + } + }, + "protocolDeviation": {"type": ["string", "null"]} + } + } + } +} diff --git a/infrastructure/aeg-arm-execution-substrate/Dockerfile b/infrastructure/aeg-arm-execution-substrate/Dockerfile new file mode 100644 index 0000000..0bebb15 --- /dev/null +++ b/infrastructure/aeg-arm-execution-substrate/Dockerfile @@ -0,0 +1,7 @@ +FROM python:3.12.11-slim-bookworm@sha256:519591d6871b7bc437060736b9f7456b8731f1499a57e22e6c285135ae657bf7 + +COPY container_worker.py /opt/aeg/container_worker.py + +WORKDIR /workspace +ENTRYPOINT ["/usr/bin/env", "-i", "HOME=/nonexistent", "LANG=C.UTF-8", "LC_ALL=C.UTF-8", "PATH=/usr/local/bin:/usr/bin:/bin", "PYTHONDONTWRITEBYTECODE=1", "PYTHONHASHSEED=0", "python3", "/opt/aeg/container_worker.py"] +CMD ["hold"] diff --git a/infrastructure/aeg-arm-execution-substrate/README.md b/infrastructure/aeg-arm-execution-substrate/README.md new file mode 100644 index 0000000..5dcc103 --- /dev/null +++ b/infrastructure/aeg-arm-execution-substrate/README.md @@ -0,0 +1,62 @@ +# AEG Arm Execution Substrate v1 + +This infrastructure runs an agent model on a GitHub Actions host while every +model-requested file or command operation executes in one locked, networkless +repair container. It is reusable laboratory infrastructure; it is not a +Situated Experience Benchmark result. + +The host controller alone receives `OPENAI_API_KEY`. It sends strict function +tools to the Responses API, validates every tool request, and invokes only the +container worker through `docker exec` with an explicit `env -i` environment +containing only inert path, locale, home, and Python-runtime settings. +The repair container receives one `arm.json`, one task directory, no controller +checkout, no hidden evaluator files, no Git remote, no socket, and no secret. +The controller copies those two inputs into a dedicated size-limited tmpfs; +the container has zero host bind mounts. + +The pinned base image is: + +```text +python:3.12.11-slim-bookworm@sha256:519591d6871b7bc437060736b9f7456b8731f1499a57e22e6c285135ae657bf7 +``` + +The workflow uses `ubuntu-24.04`; each result records the hosted runner's +runtime `ImageOS` and `ImageVersion`. Docker runs with `--network none`, a +read-only root, isolated tmpfs, all capabilities dropped, no-new-privileges, +private PID namespace, and CPU, memory, PID, workspace, file, command, token, +cost, and wall-time limits from `policy.json`. A tool timeout kills the complete +repair container rather than leaving an over-time subprocess alive. + +The host replays every Responses API output item in stateless mode so opaque +reasoning items remain continuous across strict function calls. The controller +records API token usage and computes a running cost estimate from the pinned +pricing values in `policy.json`; either ceiling stops the arm. + +## Controller operations + +```sh +python3 infrastructure/aeg-arm-execution-substrate/controller.py validate +python3 infrastructure/aeg-arm-execution-substrate/controller.py revalidate-fixtures \ + --image aeg-arm-runner:python3.12.11-slim-bookworm-v1 \ + --output /tmp/aeg-fixture-revalidation.json +python3 infrastructure/aeg-arm-execution-substrate/controller.py canary \ + --image aeg-arm-runner:python3.12.11-slim-bookworm-v1 \ + --fixture-record /tmp/aeg-fixture-revalidation.json \ + --output /tmp/aeg-canary.json \ + --encrypted-raw-output /tmp/aeg-canary-raw.p7m +``` + +`canary` requires a host-only `OPENAI_API_KEY` and the public X.509 certificate +in `AEG_RAW_OUTPUT_CERT_PEM`. The corresponding decryption key must remain +outside the repository and Actions job. Missing either input blocks readiness. + +The canary covers controller and sibling-arm paths, host credentials and +environment inheritance, Docker socket, network, hidden evaluator inputs, +human patches, caches and transcripts, symlink/absolute/`/proc` escapes, +subprocess inheritance, resource ceilings, allowlisted patch export, repair +termination before evaluation, result-schema sanitization, live model tool use, +telemetry, and encryption. It executes no benchmark arm. + +The manual workflow defaults to `canary`. Its `execute-s1` matrix is present but +requires an exact confirmation string, runs at most one matrix job at a time, +and is intentionally not invoked by this infrastructure change. diff --git a/infrastructure/aeg-arm-execution-substrate/container_worker.py b/infrastructure/aeg-arm-execution-substrate/container_worker.py new file mode 100644 index 0000000..5a60e93 --- /dev/null +++ b/infrastructure/aeg-arm-execution-substrate/container_worker.py @@ -0,0 +1,546 @@ +#!/usr/bin/env python3 +"""Trusted in-container worker for the AEG arm execution substrate.""" + +import argparse +import base64 +import binascii +import difflib +import hashlib +import io +import json +import os +import shlex +import shutil +import socket +import subprocess +import sys +import tarfile +import time +from pathlib import Path, PurePosixPath + + +ROOT = Path("/workspace") +TASK = ROOT / "task" +ARM = ROOT / "arm.json" +ALLOWED_ENV = { + "HOME": "/nonexistent", + "LANG": "C.UTF-8", + "LC_ALL": "C.UTF-8", + "PATH": "/usr/local/bin:/usr/bin:/bin", + "PYTHONDONTWRITEBYTECODE": "1", + "PYTHONHASHSEED": "0", +} +PROTECTED_NAMES = {"ISSUE.md", "arm.json", "human.patch", "test_hidden.py"} +MAX_WORKER_OUTPUT = 262144 +BASELINE = Path("/tmp/aeg-baseline") + + +class WorkerError(RuntimeError): + pass + + +def emit(value): + rendered = json.dumps(value, sort_keys=True) + if len(rendered.encode()) > MAX_WORKER_OUTPUT: + raise WorkerError("worker output exceeds limit") + print(rendered) + + +def load_arm(): + with ARM.open(encoding="utf-8") as handle: + return json.load(handle) + + +def import_bundle(encoded, max_bytes): + try: + archive_bytes = base64.b64decode(encoded, validate=True) + except (ValueError, binascii.Error): + raise WorkerError("container bundle encoding is invalid") from None + if ROOT.exists() and any(ROOT.iterdir()): + raise WorkerError("workspace is not empty before bundle import") + ROOT.mkdir(parents=True, exist_ok=True) + seen = set() + total = 0 + try: + archive = tarfile.open(fileobj=io.BytesIO(archive_bytes), mode="r:gz") + except tarfile.TarError: + raise WorkerError("container bundle archive is invalid") from None + with archive: + for member in archive.getmembers(): + relative = PurePosixPath(member.name) + if relative.is_absolute() or ".." in relative.parts or not relative.parts: + raise WorkerError("container bundle path is invalid") + if relative.parts[0] not in {"arm.json", "task"}: + raise WorkerError("container bundle has an unexpected top-level entry") + if relative.parts[0] == "arm.json" and len(relative.parts) != 1: + raise WorkerError("arm envelope path is invalid") + if member.name in seen or member.issym() or member.islnk() or member.isdev(): + raise WorkerError("container bundle entry type is prohibited") + if not member.isdir() and not member.isfile(): + raise WorkerError("container bundle entry is not regular") + seen.add(member.name) + total += member.size if member.isfile() else 0 + if total > max_bytes: + raise WorkerError("container bundle exceeds workspace limit") + destination = ROOT.joinpath(*relative.parts) + if member.isdir(): + destination.mkdir(parents=True, exist_ok=True, mode=0o755) + continue + destination.parent.mkdir(parents=True, exist_ok=True, mode=0o755) + source = archive.extractfile(member) + if source is None: + raise WorkerError("container bundle file has no payload") + with destination.open("wb") as handle: + shutil.copyfileobj(source, handle) + destination.chmod(0o644) + if {path.name for path in ROOT.iterdir()} != {"arm.json", "task"} or not ARM.is_file() or not TASK.is_dir(): + raise WorkerError("container bundle entries differ from the allowlist") + return {"files": sum(1 for path in TASK.rglob("*") if path.is_file()), "bytes": total} + + +def relative_path(value, allow_directory=False): + candidate = Path(value) + if candidate.is_absolute() or ".." in candidate.parts or ".git" in candidate.parts: + raise WorkerError("path is outside the task allowlist") + task_root = TASK.resolve(strict=True) + target = task_root / candidate + try: + resolved = target.resolve(strict=True) + except (FileNotFoundError, RuntimeError): + raise WorkerError("path does not exist") from None + if resolved != task_root and task_root not in resolved.parents: + raise WorkerError("path escapes the task workspace") + current = task_root + for part in candidate.parts: + current = current / part + if current.is_symlink(): + raise WorkerError("symlink paths are prohibited") + if resolved.is_dir() and not allow_directory: + raise WorkerError("path is not a regular file") + return resolved + + +def protected(path): + relative = path.resolve(strict=True).relative_to(TASK.resolve(strict=True)).as_posix() + name = Path(relative).name + return name in PROTECTED_NAMES or name.startswith("test") or "/test" in relative + + +def inspect_path(value, limit): + target = relative_path(value, allow_directory=True) + if target.is_dir(): + entries = [] + for child in sorted(target.iterdir(), key=lambda item: item.name): + if child.name == ".git" or child.is_symlink(): + continue + entries.append({"name": child.name, "type": "directory" if child.is_dir() else "file"}) + return {"path": target.relative_to(TASK.resolve()).as_posix() or ".", "entries": entries} + if target.stat().st_size > limit: + raise WorkerError("file exceeds inspection limit") + return { + "path": target.relative_to(TASK.resolve()).as_posix(), + "content": target.read_text(encoding="utf-8", errors="replace"), + } + + +def run_registered(argv, timeout): + arm = load_arm() + registered = shlex.split(arm["public_test_command"]) + if argv not in (registered, ["python3", "--version"]): + raise WorkerError("command is not allowlisted") + result = subprocess.run( + argv, + cwd=TASK, + env=dict(ALLOWED_ENV), + text=True, + capture_output=True, + check=False, + timeout=timeout, + ) + return { + "argv": argv, + "exit_code": result.returncode, + "stdout": result.stdout[-65536:], + "stderr": result.stderr[-65536:], + } + + +def normalize_patch_path(header, prefix): + value = header.split("\t", 1)[0].strip() + if not value.startswith(prefix): + raise WorkerError("patch header prefix is invalid") + relative = value[len(prefix):] + path = relative_path(relative) + if protected(path): + raise WorkerError("patch targets a protected input") + return path + + +def apply_hunks(original, hunks): + source = original.splitlines() + trailing_newline = original.endswith("\n") + output = [] + cursor = 0 + for old_start, lines in hunks: + old_sequence = [line[1:] for line in lines if line and line[0] in {" ", "-"}] + expected = max(old_start - 1, 0) + if source[expected:expected + len(old_sequence)] == old_sequence and expected >= cursor: + start = expected + else: + candidates = [ + index + for index in range(cursor, len(source) - len(old_sequence) + 1) + if source[index:index + len(old_sequence)] == old_sequence + ] + if len(candidates) != 1: + raise WorkerError("patch hunk context is absent or ambiguous") + start = candidates[0] + if start < cursor: + raise WorkerError("patch hunks overlap") + output.extend(source[cursor:start]) + cursor = start + for line in lines: + if not line: + raise WorkerError("patch hunk line is empty") + marker, value = line[0], line[1:] + if marker == " ": + if cursor >= len(source) or source[cursor] != value: + raise WorkerError("patch context does not match") + output.append(source[cursor]) + cursor += 1 + elif marker == "-": + if cursor >= len(source) or source[cursor] != value: + raise WorkerError("patch deletion does not match") + cursor += 1 + elif marker == "+": + output.append(value) + elif marker == "\\": + continue + else: + raise WorkerError("patch hunk marker is invalid") + output.extend(source[cursor:]) + rendered = "\n".join(output) + if trailing_newline: + rendered += "\n" + return rendered + + +def parse_and_apply_patch(patch_text, max_bytes): + if not patch_text or len(patch_text.encode()) > max_bytes: + raise WorkerError("patch is empty or exceeds limit") + lines = patch_text.splitlines() + index = 0 + changes = [] + changed_paths = set() + while index < len(lines): + if lines[index].startswith("diff --git "): + index += 1 + if index < len(lines) and lines[index].startswith("index "): + index += 1 + continue + if not lines[index].startswith("--- "): + raise WorkerError("only unified file patches are supported") + old_path = normalize_patch_path(lines[index][4:], "a/") + index += 1 + if index >= len(lines) or not lines[index].startswith("+++ "): + raise WorkerError("patch is missing new-file header") + new_path = normalize_patch_path(lines[index][4:], "b/") + if old_path != new_path: + raise WorkerError("renames and file creation are prohibited") + if old_path in changed_paths: + raise WorkerError("a file may appear only once per patch") + changed_paths.add(old_path) + index += 1 + hunks = [] + while index < len(lines) and not lines[index].startswith(("--- ", "diff --git ")): + header = lines[index] + if not header.startswith("@@ "): + raise WorkerError("patch is missing hunk header") + try: + old_field = header.split("@@", 2)[1].strip().split()[0] + old_start = int(old_field[1:].split(",", 1)[0]) + except (IndexError, ValueError): + raise WorkerError("patch hunk header is invalid") from None + index += 1 + hunk_lines = [] + while index < len(lines) and not lines[index].startswith(("@@ ", "--- ", "diff --git ")): + hunk_lines.append(lines[index]) + index += 1 + hunks.append((old_start, hunk_lines)) + original = old_path.read_text(encoding="utf-8") + rendered = apply_hunks(original, hunks) + changes.append((old_path, rendered)) + if not changes: + raise WorkerError("patch contains no file changes") + for path, rendered in changes: + path.write_text(rendered, encoding="utf-8") + return [path.relative_to(TASK.resolve()).as_posix() for path, _ in changes] + + +def workspace_snapshot(): + digest = hashlib.sha256() + files = [] + total = 0 + for path in sorted(TASK.rglob("*")): + if not path.is_file() or ".git" in path.parts or path.is_symlink(): + continue + relative = path.relative_to(TASK).as_posix() + data = path.read_bytes() + total += len(data) + files.append(relative) + digest.update(relative.encode() + b"\0" + data) + return {"sha256": digest.hexdigest(), "files": files, "bytes": total} + + +def workspace_files(root): + values = {} + for path in sorted(root.rglob("*")): + relative = path.relative_to(root) + if ".git" in relative.parts or "__pycache__" in relative.parts: + continue + if path.is_symlink(): + raise WorkerError("workspace symlinks are prohibited") + if path.is_file(): + values[relative.as_posix()] = path + return values + + +def create_baseline(): + if BASELINE.exists(): + raise WorkerError("workspace baseline already exists") + BASELINE.mkdir(mode=0o700) + for relative, source in workspace_files(TASK).items(): + destination = BASELINE / relative + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(source, destination) + return {"files": len(workspace_files(BASELINE)), "sha256": workspace_snapshot()["sha256"]} + + +def export_workspace_patch(max_bytes): + if not BASELINE.is_dir(): + raise WorkerError("workspace baseline is unavailable") + before = workspace_files(BASELINE) + after = workspace_files(TASK) + if set(before) != set(after): + raise WorkerError("file creation and deletion are prohibited") + changed = [] + diffs = [] + for relative in sorted(before): + if before[relative].read_bytes() == after[relative].read_bytes(): + continue + target = after[relative] + if protected(target): + raise WorkerError("repair changed a protected input") + try: + original = before[relative].read_text(encoding="utf-8") + repaired = target.read_text(encoding="utf-8") + except UnicodeDecodeError: + raise WorkerError("binary changes are prohibited") from None + lines = difflib.unified_diff( + original.splitlines(), + repaired.splitlines(), + fromfile=f"a/{relative}", + tofile=f"b/{relative}", + lineterm="", + ) + rendered = "\n".join(lines) + "\n" + changed.append(relative) + diffs.append(rendered) + patch = "".join(diffs) + if len(patch.encode()) > max_bytes: + raise WorkerError("exported patch exceeds limit") + return {"patch": patch, "files_changed": changed} + + +def inaccessible(path_text): + path = Path(path_text) + try: + if path.is_dir(): + next(path.iterdir(), None) + else: + path.read_bytes() + except (OSError, PermissionError): + return True + return not path.exists() + + +def canary_probe(forbidden_paths): + attempts = [] + + def add(identifier, passed, reason): + attempts.append({"id": identifier, "passed": bool(passed), "reason": reason}) + + controller_paths = [path for path in forbidden_paths if "controller" in path or "github" in path] + other_paths = [path for path in forbidden_paths if "other" in path] + add("controller_files", all(inaccessible(path) for path in controller_paths), "controller paths are absent from the container namespace") + secret_names = {name for name in os.environ if any(term in name.upper() for term in ("OPENAI", "API_KEY", "TOKEN", "SECRET", "PASSWORD", "ACTIONS_", "GITHUB_"))} + add("api_key", not any("OPENAI" in name.upper() or "API_KEY" in name.upper() for name in secret_names), "no model credential name is present") + add("github_token", not any("GITHUB" in name.upper() or "ACTIONS_" in name.upper() for name in secret_names), "no GitHub or Actions credential name is present") + add("environment_allowlist", os.environ == ALLOWED_ENV, "tool process environment equals the explicit allowlist") + add("other_arm", all(inaccessible(path) for path in other_paths), "another synthetic arm is absent from the container namespace") + add("docker_socket", not Path("/var/run/docker.sock").exists(), "Docker socket is not mounted") + tcp_failed = False + try: + with socket.create_connection(("1.1.1.1", 443), timeout=1): + pass + except OSError: + tcp_failed = True + add("network_tcp", tcp_failed, "network namespace has no external route") + dns_failed = False + try: + socket.getaddrinfo("example.com", 443) + except OSError: + dns_failed = True + add("network_dns", dns_failed, "DNS is unavailable in the networkless container") + add("hidden_tests", not any(path.name == "test_hidden.py" for path in ROOT.rglob("*")), "hidden tests are absent") + add("human_patch", not any(path.name == "human.patch" for path in ROOT.rglob("*")), "human patches are absent") + cache_targets = [Path("/root/.cache"), Path("/home/runner"), TASK / ".cache", TASK / "prior-transcript.jsonl"] + add("prior_cache_or_transcript", all(inaccessible(str(path)) for path in cache_targets), "prior cache and transcript locations are absent") + conversation_targets = [TASK / "prior-model-conversation.json", TASK / ".conversation", Path("/tmp/prior-model-conversation.json")] + add("prior_model_conversation", all(inaccessible(str(path)) for path in conversation_targets), "prior model conversation state is absent") + symlink = TASK / "canary-controller-link" + try: + symlink.symlink_to("/github/workspace/controller-sentinel") + symlink_failed = inaccessible(str(symlink)) + finally: + try: + symlink.unlink() + except OSError: + pass + add("symlink_escape", symlink_failed, "absolute symlink resolves only inside the container namespace") + add("absolute_path_escape", inaccessible("/github/workspace/controller-sentinel"), "host absolute paths are absent") + add("proc_escape", inaccessible("/proc/1/root/github/workspace/controller-sentinel"), "PID 1 root is the container root, not the host") + child = subprocess.run( + ["/usr/bin/env"], + env=dict(ALLOWED_ENV), + text=True, + capture_output=True, + check=False, + timeout=5, + ) + child_names = {line.split("=", 1)[0] for line in child.stdout.splitlines() if "=" in line} + add("subprocess_inheritance", child_names == set(ALLOWED_ENV), "child processes inherit only the explicit allowlist") + return attempts + + +def stress_pids(count): + children = [] + blocked = False + try: + for _ in range(count): + try: + children.append(subprocess.Popen(["python3", "-c", "import time; time.sleep(10)"], env=dict(ALLOWED_ENV))) + except OSError: + blocked = True + break + finally: + for child in children: + child.terminate() + for child in children: + try: + child.wait(timeout=2) + except subprocess.TimeoutExpired: + child.kill() + child.wait() + return {"requested": count, "started": len(children), "blocked": blocked} + + +def stress_disk(byte_count): + target = TASK / "canary-disk-fill" + written = 0 + blocked = False + try: + with target.open("wb") as handle: + chunk = b"0" * 1048576 + while written < byte_count: + amount = min(len(chunk), byte_count - written) + handle.write(chunk[:amount]) + handle.flush() + written += amount + os.fsync(handle.fileno()) + except OSError: + blocked = True + finally: + target.unlink(missing_ok=True) + return {"requested": byte_count, "written": written, "blocked": blocked} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest="action", required=True) + sub.add_parser("hold") + bundle = sub.add_parser("import-bundle") + bundle.add_argument("--max-bytes", required=True, type=int) + inspect = sub.add_parser("inspect") + inspect.add_argument("--path", required=True) + inspect.add_argument("--limit", required=True, type=int) + command = sub.add_parser("run") + command.add_argument("--argv-json", required=True) + command.add_argument("--timeout", required=True, type=int) + visible = sub.add_parser("visible-test") + visible.add_argument("--timeout", required=True, type=int) + patch = sub.add_parser("apply-patch") + patch.add_argument("--max-bytes", required=True, type=int) + sub.add_parser("snapshot") + sub.add_parser("baseline-create") + export = sub.add_parser("export-patch") + export.add_argument("--max-bytes", required=True, type=int) + canary = sub.add_parser("canary-probe") + canary.add_argument("--forbidden-json", required=True) + pids = sub.add_parser("stress-pids") + pids.add_argument("--count", type=int, required=True) + memory = sub.add_parser("stress-memory") + memory.add_argument("--bytes", type=int, required=True) + disk = sub.add_parser("stress-disk") + disk.add_argument("--bytes", type=int, required=True) + sleeper = sub.add_parser("sleep") + sleeper.add_argument("--seconds", type=float, required=True) + args = parser.parse_args() + + if args.action == "hold": + while True: + time.sleep(60) + if args.action == "import-bundle": + result = import_bundle(sys.stdin.read(), args.max_bytes) + elif args.action == "inspect": + result = inspect_path(args.path, args.limit) + elif args.action == "run": + argv = json.loads(args.argv_json) + if not isinstance(argv, list) or not all(isinstance(item, str) for item in argv): + raise WorkerError("command must be a string array") + result = run_registered(argv, args.timeout) + elif args.action == "visible-test": + result = run_registered(shlex.split(load_arm()["public_test_command"]), args.timeout) + elif args.action == "apply-patch": + result = {"files_changed": parse_and_apply_patch(sys.stdin.read(), args.max_bytes)} + elif args.action == "snapshot": + result = workspace_snapshot() + elif args.action == "baseline-create": + result = create_baseline() + elif args.action == "export-patch": + result = export_workspace_patch(args.max_bytes) + elif args.action == "canary-probe": + forbidden = json.loads(args.forbidden_json) + result = {"attempts": canary_probe(forbidden)} + elif args.action == "stress-pids": + result = stress_pids(args.count) + elif args.action == "stress-memory": + value = bytearray(args.bytes) + result = {"allocated": len(value)} + elif args.action == "stress-disk": + result = stress_disk(args.bytes) + elif args.action == "sleep": + time.sleep(args.seconds) + result = {"slept": args.seconds} + else: + raise WorkerError("unknown action") + emit(result) + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except (WorkerError, json.JSONDecodeError, subprocess.TimeoutExpired) as error: + print(f"container worker error: {error}", file=sys.stderr) + raise SystemExit(2) diff --git a/infrastructure/aeg-arm-execution-substrate/controller.py b/infrastructure/aeg-arm-execution-substrate/controller.py new file mode 100644 index 0000000..5729056 --- /dev/null +++ b/infrastructure/aeg-arm-execution-substrate/controller.py @@ -0,0 +1,1169 @@ +#!/usr/bin/env python3 +"""Host-only controller for disposable AEG repair and evaluator containers.""" + +import argparse +import base64 +import hashlib +import importlib.util +import io +import json +import os +import re +import shlex +import shutil +import subprocess +import sys +import tarfile +import tempfile +import time +import urllib.error +import urllib.request +import uuid +from pathlib import Path + +from jsonschema import Draft202012Validator + + +HERE = Path(__file__).resolve().parent +REPO = HERE.parents[1] +BENCH = REPO / "experiments" / "situated-experience-benchmark-v1" +MANIFEST = BENCH / "s1-manifest.json" +PLAN = BENCH / "execution" / "s1-execution-plan.json" +POLICY = HERE / "policy.json" +CANARY_SCHEMA = HERE / "schemas" / "canary-result.schema.json" +ARM_RESULT_SCHEMA = BENCH / "schemas" / "arm-result.schema.json" +AGENT_RESULT_SCHEMA = BENCH / "schemas" / "agent-result.schema.json" +WORKFLOW = REPO / ".github" / "workflows" / "aeg-arm-execution-substrate.yml" +OPENAI_ENDPOINT = "https://api.openai.com/v1/responses" + + +class ControllerError(RuntimeError): + pass + + +def load_json(path): + with Path(path).open(encoding="utf-8") as handle: + return json.load(handle) + + +def write_json(path, value): + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + +def sha256_file(path): + return hashlib.sha256(Path(path).read_bytes()).hexdigest() + + +def schema_validate(instance, schema_path, label): + validator = Draft202012Validator(load_json(schema_path)) + errors = sorted(validator.iter_errors(instance), key=lambda item: list(item.path)) + if errors: + rendered = "; ".join(f"{'.'.join(map(str, error.path)) or ''}: {error.message}" for error in errors) + raise ControllerError(f"{label} schema invalid: {rendered}") + + +def scrubbed_host_environment(): + sensitive_fragments = ("_API_KEY", "_TOKEN", "_SECRET", "_PASSWORD", "CREDENTIAL") + return { + name: value + for name, value in os.environ.items() + if name not in {"OPENAI_API_KEY", "GITHUB_TOKEN", "AEG_RAW_OUTPUT_CERT_PEM"} + and not name.startswith("ACTIONS_") + and not any(fragment in name.upper() for fragment in sensitive_fragments) + } + + +def run(args, cwd=None, input_text=None, timeout=120, check=True, env=None): + result = subprocess.run( + args, + cwd=cwd, + input=input_text, + env=scrubbed_host_environment() if env is None else env, + text=True, + capture_output=True, + timeout=timeout, + check=False, + ) + if check and result.returncode: + raise ControllerError(result.stderr.strip() or result.stdout.strip() or f"command failed: {args[0]}") + return result + + +def encoded_sanitized_bundle(task_root, max_bytes): + task_root = Path(task_root) + entries = [task_root / "arm.json", task_root / "task"] + entries.extend(sorted((task_root / "task").rglob("*"))) + total = 0 + buffer = io.BytesIO() + with tarfile.open(fileobj=buffer, mode="w:gz", format=tarfile.PAX_FORMAT) as archive: + for path in entries: + if path.is_symlink(): + raise ControllerError("container bundle contains a symlink") + if not path.is_dir() and not path.is_file(): + raise ControllerError("container bundle contains a non-regular entry") + if path.is_file(): + total += path.stat().st_size + if total > max_bytes: + raise ControllerError("container bundle exceeds the workspace ceiling") + relative = path.relative_to(task_root).as_posix() + info = archive.gettarinfo(str(path), arcname=relative) + info.uid = info.gid = 0 + info.uname = info.gname = "" + info.mtime = 0 + info.mode = 0o755 if path.is_dir() else 0o644 + if path.is_file(): + with path.open("rb") as handle: + archive.addfile(info, handle) + else: + archive.addfile(info) + return base64.b64encode(buffer.getvalue()).decode("ascii") + + +def expected_arm_ids(manifest): + values = [] + for pair in manifest["pairs"]: + for replicate, order in enumerate(manifest["protocol"]["arm_orders"][pair["pair_id"]], 1): + for mode in order: + values.append(f"{pair['pair_id']}--r{replicate:02d}--{mode}") + return values + + +def validate_configuration(): + policy = load_json(POLICY) + manifest = load_json(MANIFEST) + plan = load_json(PLAN) + expected_manifest = policy["benchmark"]["manifest_sha256"] + expected_plan = policy["benchmark"]["execution_plan_sha256"] + if sha256_file(MANIFEST) != expected_manifest: + raise ControllerError("frozen manifest hash differs from substrate policy") + if sha256_file(PLAN) != expected_plan: + raise ControllerError("frozen execution-plan hash differs from substrate policy") + if manifest["protocol"]["model"] != policy["benchmark"]["model"]: + raise ControllerError("model differs from frozen manifest") + if policy["bridge"]["max_model_turns"] <= manifest["protocol"]["budget"]["max_completed_commands"]: + raise ControllerError("model-turn safety ceiling would reduce the frozen command budget") + plan_ids = [item["arm_id"] for item in plan["arms"]] + if plan_ids != expected_arm_ids(manifest) or plan["arm_count"] != 12: + raise ControllerError("tracked execution plan differs from frozen arm order") + dockerfile = (HERE / "Dockerfile").read_text(encoding="utf-8") + if f"FROM {policy['base_image']}" not in dockerfile: + raise ControllerError("Dockerfile base image is not the pinned policy image") + if WORKFLOW.exists(): + workflow_text = WORKFLOW.read_text(encoding="utf-8") + workflow_coordinates = re.findall( + r"- sequence:\s*(\d+)\s+arm_id:\s*(s1-[a-z0-9-]+--r\d{2}--(?:control|aeg-assisted))", + workflow_text, + ) + if workflow_coordinates != [(str(index), arm_id) for index, arm_id in enumerate(plan_ids, 1)]: + raise ControllerError("workflow matrix differs from the frozen arm order") + result = run(["python3", str(BENCH / "run_benchmark.py"), "validate"], cwd=REPO) + return { + "status": "valid", + "substrate_id": policy["substrate_id"], + "manifest_sha256": expected_manifest, + "execution_plan_sha256": expected_plan, + "arms": len(plan_ids), + "benchmark_validation": json.loads(result.stdout)["status"], + } + + +class BudgetGuard: + def __init__(self, policy, arm_budget): + self.policy = policy + self.arm_budget = arm_budget + self.started = time.monotonic() + self.commands = 0 + self.tool_turns = 0 + self.input_tokens = 0 + self.output_tokens = 0 + self.cached_tokens = 0 + self.cost_usd = 0.0 + + def check_wall(self): + if time.monotonic() - self.started > self.arm_budget["wall_time_seconds"]: + raise ControllerError("wall-time budget exceeded") + + def add_tool_turn(self): + self.check_wall() + self.tool_turns += 1 + if self.tool_turns > self.policy["bridge"]["max_model_turns"]: + raise ControllerError("model tool-turn limit exceeded") + + def add_command(self): + self.check_wall() + self.commands += 1 + if self.commands > self.arm_budget["max_completed_commands"]: + raise ControllerError("completed-command budget exceeded") + + def add_usage(self, usage): + self.check_wall() + input_tokens = int(usage.get("input_tokens") or 0) + output_tokens = int(usage.get("output_tokens") or 0) + details = usage.get("input_tokens_details") or {} + cached = int(details.get("cached_tokens") or 0) + self.input_tokens += input_tokens + self.output_tokens += output_tokens + self.cached_tokens += min(cached, input_tokens) + uncached = max(input_tokens - cached, 0) + prices = self.policy["pricing_usd_per_million_tokens"] + self.cost_usd += ( + uncached * prices["input"] + + cached * prices["cached_input"] + + output_tokens * prices["output"] + ) / 1_000_000 + ceiling = self.policy["safety_ceiling"] + if self.input_tokens + self.output_tokens > ceiling["max_total_tokens"]: + raise ControllerError("token safety ceiling exceeded") + if self.cost_usd > ceiling["max_cost_usd"]: + raise ControllerError("cost safety ceiling exceeded") + + def check_workspace(self, size): + if size > self.policy["container"]["workspace_bytes"]: + raise ControllerError("workspace disk ceiling exceeded") + + def telemetry(self): + return { + "input_tokens": self.input_tokens, + "output_tokens": self.output_tokens, + "cached_input_tokens": self.cached_tokens, + "estimated_cost_usd": round(self.cost_usd, 8), + } + + +class DockerRuntime: + def __init__(self, image, policy): + self.image = image + self.policy = policy + + def security_args(self, task_root, name): + limits = self.policy["container"] + return [ + "docker", "run", "--detach", "--name", name, + "--network", "none", + "--read-only", + "--cap-drop", "ALL", + "--security-opt", "no-new-privileges:true", + "--pids-limit", str(limits["pids_limit"]), + "--memory", str(limits["memory_bytes"]), + "--memory-swap", str(limits["memory_bytes"]), + "--cpus", str(limits["cpus"]), + "--ulimit", "nofile=256:256", + "--ipc", "private", + "--hostname", "aeg-repair", + "--user", f"{os.getuid()}:{os.getgid()}", + "--tmpfs", f"/tmp:rw,noexec,nosuid,nodev,size={limits['tmpfs_bytes']}", + "--tmpfs", f"/workspace:rw,nosuid,nodev,size={limits['workspace_bytes']},mode=0700,uid={os.getuid()},gid={os.getgid()}", + self.image, + "hold", + ] + + def start(self, task_root, prefix="aeg-repair"): + task_root = Path(task_root) + entries = {path.name for path in task_root.iterdir()} + if entries != {"arm.json", "task"}: + raise ControllerError(f"container mount entries differ from allowlist: {sorted(entries)}") + name = f"{prefix}-{uuid.uuid4().hex[:12]}" + result = run(self.security_args(task_root, name)) + container_id = result.stdout.strip() + if not container_id: + raise ControllerError("Docker did not return a container id") + try: + payload = encoded_sanitized_bundle(task_root, self.policy["container"]["workspace_bytes"]) + self.exec( + container_id, + ["import-bundle", "--max-bytes", str(self.policy["container"]["workspace_bytes"])], + input_text=payload, + ) + self.exec(container_id, ["baseline-create"]) + except Exception: + self.remove(container_id) + raise + return container_id + + def worker_command(self, container_id, arguments): + env = self.policy["container"]["allowed_process_environment"] + assignments = [f"{key}={value}" for key, value in sorted(env.items())] + return [ + "docker", "exec", "--interactive", container_id, "/usr/bin/env", "-i", *assignments, + "python3", "/opt/aeg/container_worker.py", *arguments, + ] + + def exec(self, container_id, arguments, input_text=None, timeout=120, check=True): + try: + result = run(self.worker_command(container_id, arguments), input_text=input_text, timeout=timeout, check=check) + except subprocess.TimeoutExpired: + run(["docker", "kill", container_id], timeout=30, check=False) + raise + if check: + try: + return json.loads(result.stdout) + except json.JSONDecodeError: + raise ControllerError("container worker returned invalid JSON") from None + return result + + def security_attestation(self, container_id): + inspected = run(["docker", "inspect", container_id]) + value = json.loads(inspected.stdout)[0] + host = value["HostConfig"] + expected = self.policy["container"] + passed = ( + host.get("NetworkMode") == "none" + and host.get("PidMode") in (None, "") + and host.get("ReadonlyRootfs") is True + and set(host.get("CapDrop") or []) == {"ALL"} + and "no-new-privileges:true" in (host.get("SecurityOpt") or []) + and int(host.get("PidsLimit") or 0) == expected["pids_limit"] + and int(host.get("Memory") or 0) == expected["memory_bytes"] + and int(host.get("NanoCpus") or 0) == int(expected["cpus"] * 1_000_000_000) + and not any("docker.sock" in mount.get("Source", "") for mount in value.get("Mounts", [])) + and not value.get("Mounts") + and not host.get("Binds") + and set((host.get("Tmpfs") or {}).keys()) == {"/tmp", "/workspace"} + ) + return passed + + def image_id(self): + result = run(["docker", "image", "inspect", self.image, "--format", "{{.Id}}"]) + return result.stdout.strip() or None + + def remove(self, container_id): + run(["docker", "rm", "--force", container_id], check=False) + + +class ToolBridge: + def __init__(self, runtime, container_id, policy, arm_budget, public_test_command=None): + self.runtime = runtime + self.container_id = container_id + self.policy = policy + self.public_test_argv = shlex.split(public_test_command) if public_test_command else None + self.guard = BudgetGuard(policy, arm_budget) + self.files_inspected = set() + self.tests = [] + self.attempt_hashes = [] + self.tool_events = [] + + def _record(self, name, arguments, result): + self.tool_events.append({"tool": name, "arguments": arguments, "result": result}) + + def call(self, name, arguments): + self.guard.add_tool_turn() + allowed = set(self.policy["bridge"]["allowed_tools"]) + if name not in allowed or not isinstance(arguments, dict): + raise ControllerError("model requested an unknown tool") + if name == "inspect_file": + if set(arguments) != {"path"} or not isinstance(arguments["path"], str): + raise ControllerError("inspect_file arguments are invalid") + result = self.runtime.exec( + self.container_id, + ["inspect", "--path", arguments["path"], "--limit", str(self.policy["bridge"]["max_file_read_bytes"])], + ) + self.files_inspected.add(result["path"]) + elif name == "run_command": + if set(arguments) != {"command"} or not isinstance(arguments["command"], str): + raise ControllerError("run_command arguments are invalid") + self.guard.add_command() + argv = shlex.split(arguments["command"]) + result = self.runtime.exec( + self.container_id, + ["run", "--argv-json", json.dumps(argv), "--timeout", "120"], + timeout=125, + ) + if self.public_test_argv is not None and result.get("argv") == self.public_test_argv: + self.tests.append({"command": arguments["command"], "scope": "agent", "passed": result["exit_code"] == 0}) + elif name == "run_visible_tests": + if arguments: + raise ControllerError("run_visible_tests accepts no arguments") + self.guard.add_command() + result = self.runtime.exec(self.container_id, ["visible-test", "--timeout", "120"], timeout=125) + self.tests.append({"command": "registered visible test", "scope": "agent", "passed": result["exit_code"] == 0}) + else: + if set(arguments) != {"patch"} or not isinstance(arguments["patch"], str): + raise ControllerError("apply_patch arguments are invalid") + if len(arguments["patch"].encode()) > self.policy["bridge"]["max_patch_bytes"]: + raise ControllerError("patch exceeds bridge limit") + result = self.runtime.exec( + self.container_id, + ["apply-patch", "--max-bytes", str(self.policy["bridge"]["max_patch_bytes"])], + input_text=arguments["patch"], + ) + snapshot = self.runtime.exec(self.container_id, ["snapshot"]) + self.guard.check_workspace(snapshot["bytes"]) + if snapshot["sha256"] not in self.attempt_hashes: + if len(self.attempt_hashes) >= self.guard.arm_budget["max_attempts"]: + raise ControllerError("repair-attempt budget exceeded") + self.attempt_hashes.append(snapshot["sha256"]) + serialized = json.dumps(result, sort_keys=True) + if len(serialized.encode()) > self.policy["bridge"]["max_tool_output_bytes"]: + raise ControllerError("tool output exceeds bridge limit") + self._record(name, {key: "" if key == "patch" else value for key, value in arguments.items()}, result) + return result + + +def function_tools(): + return [ + { + "type": "function", + "name": "inspect_file", + "description": "Read one relative file or list one relative directory inside the repair task workspace.", + "strict": True, + "parameters": { + "type": "object", + "properties": {"path": {"type": "string"}}, + "required": ["path"], + "additionalProperties": False, + }, + }, + { + "type": "function", + "name": "run_command", + "description": "Run an allowlisted diagnostic or registered visible-test command inside the repair container.", + "strict": True, + "parameters": { + "type": "object", + "properties": {"command": {"type": "string"}}, + "required": ["command"], + "additionalProperties": False, + }, + }, + { + "type": "function", + "name": "apply_patch", + "description": "Apply a unified diff to existing production files inside the repair workspace. Tests and task inputs are protected.", + "strict": True, + "parameters": { + "type": "object", + "properties": {"patch": {"type": "string"}}, + "required": ["patch"], + "additionalProperties": False, + }, + }, + { + "type": "function", + "name": "run_visible_tests", + "description": "Run the envelope's registered visible test command inside the repair container.", + "strict": True, + "parameters": {"type": "object", "properties": {}, "required": [], "additionalProperties": False}, + }, + ] + + +def output_text(response): + if isinstance(response.get("output_text"), str): + return response["output_text"] + values = [] + for item in response.get("output", []): + if item.get("type") != "message": + continue + for content in item.get("content", []): + if content.get("type") == "output_text" and isinstance(content.get("text"), str): + values.append(content["text"]) + return "".join(values) + + +class ResponsesModelClient: + def __init__(self, api_key, model, policy, bridge, raw_path): + if not api_key: + raise ControllerError("host model credential is unavailable") + self.api_key = api_key + self.model = model + self.policy = policy + self.bridge = bridge + self.raw_path = Path(raw_path) + self.raw_path.parent.mkdir(parents=True, exist_ok=True) + + def request(self, payload): + request = urllib.request.Request( + OPENAI_ENDPOINT, + data=json.dumps(payload).encode(), + headers={"Authorization": f"Bearer {self.api_key}", "Content-Type": "application/json"}, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=180) as response: + return json.loads(response.read()) + except urllib.error.HTTPError as error: + detail = error.read().decode(errors="replace")[:1000] + raise ControllerError(f"model API request failed with HTTP {error.code}: {detail}") from None + except urllib.error.URLError as error: + raise ControllerError(f"model API request failed: {error.reason}") from None + + def run(self, prompt, final_schema=None, forced_first_tool=None): + conversation = [{"role": "user", "content": prompt}] + final = "" + for turn in range(self.policy["bridge"]["max_model_turns"]): + payload = { + "model": self.model, + "input": conversation, + "tools": function_tools(), + "parallel_tool_calls": False, + "store": False, + "max_output_tokens": self.policy["safety_ceiling"]["max_output_tokens_per_response"], + } + if turn == 0 and forced_first_tool: + payload["tool_choice"] = {"type": "function", "name": forced_first_tool} + if final_schema: + payload["text"] = { + "format": { + "type": "json_schema", + "name": "agent_result", + "strict": True, + "schema": final_schema, + } + } + response = self.request(payload) + self.bridge.guard.add_usage(response.get("usage") or {}) + with self.raw_path.open("a", encoding="utf-8") as handle: + handle.write(json.dumps({"kind": "model_response", "value": response}, sort_keys=True) + "\n") + calls = [item for item in response.get("output", []) if item.get("type") == "function_call"] + conversation.extend(response.get("output", [])) + if not calls: + final = output_text(response) + break + for call in calls: + try: + arguments = json.loads(call.get("arguments") or "{}") + except json.JSONDecodeError: + raise ControllerError("model returned invalid tool arguments") from None + result = self.bridge.call(call.get("name"), arguments) + with self.raw_path.open("a", encoding="utf-8") as handle: + handle.write(json.dumps({"kind": "tool_result", "call_id": call.get("call_id"), "value": result}, sort_keys=True) + "\n") + conversation.append({ + "type": "function_call_output", + "call_id": call.get("call_id"), + "output": json.dumps(result, sort_keys=True), + }) + else: + raise ControllerError("model turn limit exceeded") + if not final: + raise ControllerError("model returned no final output") + return final + + +def encrypted_raw_output(raw_path, encrypted_path, certificate_pem): + raw_path = Path(raw_path) + encrypted_path = Path(encrypted_path) + try: + if not certificate_pem or "BEGIN CERTIFICATE" not in certificate_pem: + raise ControllerError("raw-output public certificate is unavailable") + encrypted_path.parent.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(prefix="aeg-public-cert-") as raw: + cert = Path(raw) / "recipient.pem" + cert.write_text(certificate_pem, encoding="utf-8") + run([ + "openssl", "cms", "-encrypt", "-binary", "-aes-256-cbc", + "-in", str(raw_path), "-out", str(encrypted_path), "-outform", "DER", str(cert), + ]) + return encrypted_path.is_file() and encrypted_path.stat().st_size > 0 + finally: + raw_path.unlink(missing_ok=True) + + +def stage_root(source, destination, arm): + destination = Path(destination) + if destination.exists(): + raise ControllerError("container staging root already exists") + destination.mkdir(parents=True) + shutil.copyfile(arm, destination / "arm.json") + shutil.copytree(source, destination / "task") + return destination + + +def fixture_pair(manifest, pair_id): + for pair in manifest["pairs"]: + if pair["pair_id"] == pair_id: + return pair + raise ControllerError(f"unknown pair: {pair_id}") + + +def minimal_arm(command): + return {"public_test_command": command} + + +def revalidate_fixtures(image, output): + validation = validate_configuration() + policy = load_json(POLICY) + manifest = load_json(MANIFEST) + runtime = DockerRuntime(image, policy) + details = [] + failures = patches = 0 + for pair in manifest["pairs"]: + for stage in ("source", "transfer"): + stage_root_path = BENCH / "fixtures" / pair["pair_id"] / stage + seed_name = "buggy" if stage == "source" else "agent" + command = pair["source_public_test_command"] if stage == "source" else pair["public_test_command"] + pattern = pair["source_initial_failure_pattern"] if stage == "source" else pair["initial_failure_pattern"] + with tempfile.TemporaryDirectory(prefix="aeg-fixture-container-") as raw: + root = Path(raw) / "repair" + root.mkdir() + write_json(root / "arm.json", minimal_arm(command)) + shutil.copytree(stage_root_path / seed_name, root / "task") + container = runtime.start(root, prefix="aeg-fixture-buggy") + try: + initial = runtime.exec(container, ["visible-test", "--timeout", "120"], timeout=125) + combined = initial["stdout"] + "\n" + initial["stderr"] + if initial["exit_code"] == 0 or not re.search(pattern, combined, re.IGNORECASE | re.DOTALL): + raise ControllerError(f"{pair['pair_id']} {stage} did not fail for the registered reason in the pinned image") + failures += 1 + finally: + runtime.remove(container) + with tempfile.TemporaryDirectory(prefix="aeg-evaluator-container-") as raw: + root = Path(raw) / "evaluator" + root.mkdir() + write_json(root / "arm.json", minimal_arm(pair["hidden_test_command"])) + shutil.copytree(stage_root_path / seed_name, root / "task") + shutil.copyfile(stage_root_path / "evaluator" / "test_hidden.py", root / "task" / "test_hidden.py") + container = runtime.start(root, prefix="aeg-fixture-evaluator") + try: + patch_text = (stage_root_path / "evaluator" / "human.patch").read_text(encoding="utf-8") + runtime.exec(container, ["apply-patch", "--max-bytes", str(policy["bridge"]["max_patch_bytes"])], input_text=patch_text) + evaluated = runtime.exec(container, ["visible-test", "--timeout", "120"], timeout=125) + if evaluated["exit_code"] != 0: + raise ControllerError(f"{pair['pair_id']} {stage} human patch failed in the pinned evaluator container") + patches += 1 + finally: + runtime.remove(container) + details.append({"pair_id": pair["pair_id"], "stage": stage, "buggy_failure_matched": True, "human_patch_passed": True}) + record = { + "schema_version": "1.0.0", + "substrate_id": policy["substrate_id"], + "manifest_sha256": validation["manifest_sha256"], + "base_image": policy["base_image"], + "runtime_image_id": runtime.image_id(), + "buggy_failures": failures, + "human_patches_passed": patches, + "details": details, + } + write_json(output, record) + return record + + +def canary_arm_root(raw, mode): + root = Path(raw) / f"canary-{mode}" + root.mkdir() + write_json(root / "arm.json", minimal_arm("python3 --version")) + task = root / "task" + task.mkdir() + (task / "CANARY.txt").write_text("non-benchmark substrate canary\n", encoding="utf-8") + return root + + +def security_canary(runtime, policy, root, forbidden_paths): + container = runtime.start(root, prefix="aeg-security-canary") + try: + attested = runtime.security_attestation(container) + probe = runtime.exec(container, ["canary-probe", "--forbidden-json", json.dumps(forbidden_paths)]) + attempts = probe["attempts"] + attempts.append({"id": "container_security_configuration", "passed": attested, "reason": "Docker inspect matches the frozen security policy"}) + pids = runtime.exec(container, ["stress-pids", "--count", "96"], timeout=30) + attempts.append({"id": "process_limit", "passed": bool(pids["blocked"] and pids["started"] < 96), "reason": "pids-limit rejected excess child processes"}) + memory = runtime.exec(container, ["stress-memory", "--bytes", str(policy["container"]["memory_bytes"] + 268435456)], timeout=30, check=False) + attempts.append({"id": "memory_limit", "passed": memory.returncode != 0, "reason": "memory cgroup rejected an over-limit allocation"}) + disk = runtime.exec(container, ["stress-disk", "--bytes", str(policy["container"]["workspace_bytes"] * 2)], timeout=30) + attempts.append({"id": "workspace_tmpfs_limit", "passed": bool(disk["blocked"] and disk["written"] < disk["requested"]), "reason": "workspace tmpfs rejected a write above its hard size limit"}) + wall_blocked = False + try: + runtime.exec(container, ["sleep", "--seconds", "5"], timeout=1, check=False) + except subprocess.TimeoutExpired: + wall_blocked = True + attempts.append({"id": "wall_time_limit", "passed": wall_blocked, "reason": "controller timeout killed the over-time repair container"}) + arm_budget = {"wall_time_seconds": 900, "max_completed_commands": 40, "max_attempts": 3} + guard = BudgetGuard(policy, arm_budget) + command_passed = False + try: + for _ in range(41): + guard.add_command() + except ControllerError: + command_passed = True + attempts.append({"id": "command_limit", "passed": command_passed, "reason": "controller rejected command 41"}) + token_guard = BudgetGuard(policy, arm_budget) + token_passed = False + try: + token_guard.add_usage({"input_tokens": policy["safety_ceiling"]["max_total_tokens"] + 1, "output_tokens": 0}) + except ControllerError: + token_passed = True + attempts.append({"id": "token_limit", "passed": token_passed, "reason": "controller rejected usage above the token ceiling"}) + cost_guard = BudgetGuard(policy, arm_budget) + cost_passed = False + try: + cost_guard.add_usage({"input_tokens": 0, "output_tokens": 2_000_000}) + except ControllerError: + cost_passed = True + attempts.append({"id": "cost_limit", "passed": cost_passed, "reason": "controller rejected usage above the cost ceiling"}) + disk_passed = False + try: + guard.check_workspace(policy["container"]["workspace_bytes"] + 1) + except ControllerError: + disk_passed = True + attempts.append({"id": "disk_limit", "passed": disk_passed, "reason": "controller rejected a workspace above the disk ceiling"}) + return attempts, attested + finally: + runtime.remove(container) + + +def canary_sanitized_result(): + return { + "schema_version": "1.0.0", + "benchmark_id": "situated-experience-benchmark-v1", + "family": "S1", + "pair_id": "non-benchmark-substrate-canary", + "replicate": 1, + "mode": "control", + "evaluation_status": "evaluated", + "input_hashes": {"arm": "0" * 64, "task": "1" * 64, "manifest": "2" * 64}, + "budget": {"wall_time_seconds": 60, "max_completed_commands": 4, "max_attempts": 1}, + "regression_free_success": True, + "attempts": 1, + "completed_commands": 1, + "tests_run": [{"command": "non-benchmark evaluator canary", "scope": "hidden", "passed": True}], + "files_inspected": ["module.py"], + "files_changed": ["module.py"], + "patch_size": {"added_lines": 1, "deleted_lines": 1, "files": 1}, + "wall_time_ms": 1, + "tokens": {"input": 0, "output": 0, "unavailable_reason": None}, + "failed_historical_paths_repeated": [], + "environment_assumptions_checked": [{"assumption": "canary only", "checked": True, "evidence": "separate evaluator passed"}], + "experiences": [{"experience_id": None, "disposition": "abstained", "reason": "non-benchmark canary"}], + "negative_transfer": None, + "evaluator_findings": ["non-benchmark hidden evaluator passed after repair termination"], + "limitations": ["synthetic substrate canary; not a benchmark arm or outcome"], + } + + +def functional_boundary_canary(runtime, policy, raw): + attempts = [] + repair_root = Path(raw) / "functional-repair" + repair_root.mkdir() + write_json(repair_root / "arm.json", minimal_arm("python3 --version")) + task = repair_root / "task" + task.mkdir() + (task / "module.py").write_text("VALUE = 'old'\n", encoding="utf-8") + (task / "ISSUE.md").write_text("protected canary input\n", encoding="utf-8") + repair = runtime.start(repair_root, prefix="aeg-functional-repair") + exported = None + try: + bridge = ToolBridge( + runtime, + repair, + policy, + {"wall_time_seconds": 60, "max_completed_commands": 4, "max_attempts": 1}, + public_test_command="python3 --version", + ) + bridge.call("inspect_file", {"path": "module.py"}) + bridge.call("apply_patch", {"patch": "--- a/module.py\n+++ b/module.py\n@@ -1 +1 @@\n-VALUE = 'old'\n+VALUE = 'new'\n"}) + bridge.call("run_visible_tests", {}) + exported = runtime.exec( + repair, + ["export-patch", "--max-bytes", str(policy["bridge"]["max_patch_bytes"])], + ) + finally: + runtime.remove(repair) + patch_ok = ( + exported is not None + and exported["files_changed"] == ["module.py"] + and "+VALUE = 'new'" in exported["patch"] + and "ISSUE.md" not in exported["patch"] + ) + attempts.append({"id": "patch_export_allowlist", "passed": patch_ok, "reason": "only the bridge-modified production file was exported"}) + terminated = run(["docker", "inspect", repair], check=False).returncode != 0 + + evaluator_root = Path(raw) / "functional-evaluator" + evaluator_root.mkdir() + write_json(evaluator_root / "arm.json", minimal_arm("python3 -m unittest -v test_hidden.py")) + evaluator_task = evaluator_root / "task" + evaluator_task.mkdir() + (evaluator_task / "module.py").write_text("VALUE = 'new'\n", encoding="utf-8") + (evaluator_task / "test_hidden.py").write_text( + "import unittest\nimport module\n\nclass HiddenCanary(unittest.TestCase):\n def test_value(self):\n self.assertEqual(module.VALUE, 'new')\n", + encoding="utf-8", + ) + evaluator = runtime.start(evaluator_root, prefix="aeg-functional-evaluator") + try: + evaluated = runtime.exec(evaluator, ["visible-test", "--timeout", "30"], timeout=35) + finally: + runtime.remove(evaluator) + evaluator_ok = terminated and evaluator != repair and evaluated["exit_code"] == 0 + attempts.append({"id": "repair_termination_before_evaluator", "passed": evaluator_ok, "reason": "repair container was absent before the distinct hidden evaluator started"}) + + schema_ok = True + try: + schema_validate(canary_sanitized_result(), ARM_RESULT_SCHEMA, "canary sanitized result") + except ControllerError: + schema_ok = False + attempts.append({"id": "sanitizer_measurement_schema", "passed": schema_ok, "reason": "approved canary metrics validate against the frozen arm-result schema"}) + return attempts + + +def live_model_canary(runtime, policy, api_key, raw_path): + aggregate = {"input_tokens": 0, "output_tokens": 0, "estimated_cost_usd": 0.0} + modes = {} + tool_call_in_container = True + with tempfile.TemporaryDirectory(prefix="aeg-live-model-canary-") as raw: + for mode in ("control", "treatment"): + root = canary_arm_root(raw, mode) + container = runtime.start(root, prefix=f"aeg-model-{mode}") + try: + bridge = ToolBridge( + runtime, + container, + policy, + {"wall_time_seconds": 180, "max_completed_commands": 40, "max_attempts": 3}, + public_test_command="python3 --version", + ) + client = ResponsesModelClient(api_key, policy["benchmark"]["model"], policy, bridge, raw_path) + prompt = ( + "This is a non-benchmark security canary. Use inspect_file on CANARY.txt exactly once, " + f"then report that the {mode} telemetry canary completed. Do not propose or apply a repair." + ) + client.run(prompt, forced_first_tool="inspect_file") + telemetry = bridge.guard.telemetry() + modes[mode] = telemetry["input_tokens"] > 0 and telemetry["output_tokens"] > 0 + aggregate["input_tokens"] += telemetry["input_tokens"] + aggregate["output_tokens"] += telemetry["output_tokens"] + aggregate["estimated_cost_usd"] += telemetry["estimated_cost_usd"] + tool_call_in_container = tool_call_in_container and any(event["tool"] == "inspect_file" for event in bridge.tool_events) + finally: + runtime.remove(container) + aggregate["estimated_cost_usd"] = round(aggregate["estimated_cost_usd"], 8) + return modes, tool_call_in_container, aggregate + + +def canary(image, fixture_record_path, output, encrypted_output): + validate_configuration() + policy = load_json(POLICY) + fixture = load_json(fixture_record_path) + runtime = DockerRuntime(image, policy) + credential_present = bool(os.environ.get("OPENAI_API_KEY")) + certificate = os.environ.get(policy["raw_output"]["public_certificate_environment"], "") + raw_path = Path(tempfile.mkdtemp(prefix="aeg-canary-private-")) / "raw.jsonl" + raw_path.write_text("", encoding="utf-8") + with tempfile.TemporaryDirectory(prefix="aeg-substrate-canary-") as raw: + host_private = Path(raw) / "controller-private" + other_arm = Path(raw) / "other-synthetic-arm" + host_private.mkdir() + other_arm.mkdir() + (host_private / "controller-sentinel").write_text("controller-only\n", encoding="utf-8") + (host_private / "human.patch").write_text("controller-only\n", encoding="utf-8") + (host_private / "test_hidden.py").write_text("controller-only\n", encoding="utf-8") + (host_private / "prior-transcript.jsonl").write_text("controller-only\n", encoding="utf-8") + (other_arm / "sentinel").write_text("other-arm-only\n", encoding="utf-8") + probe_root = canary_arm_root(raw, "security") + attempts, security_matches = security_canary( + runtime, + policy, + probe_root, + [str(host_private), str(other_arm), "/github/workspace/controller-sentinel"], + ) + attempts.extend(functional_boundary_canary(runtime, policy, raw)) + modes = {"control": False, "treatment": False} + tool_inside = False + telemetry = {"input_tokens": 0, "output_tokens": 0, "estimated_cost_usd": 0.0} + boundary_errors = [] + if credential_present: + try: + modes, tool_inside, telemetry = live_model_canary(runtime, policy, os.environ["OPENAI_API_KEY"], raw_path) + except ControllerError as error: + boundary_errors.append(str(error)) + else: + boundary_errors.append("host model credential is unavailable") + encrypted = False + if raw_path.exists(): + try: + encrypted = encrypted_raw_output(raw_path, encrypted_output, certificate) + except ControllerError as error: + boundary_errors.append(str(error)) + fixture_passed = fixture.get("buggy_failures") == 4 and fixture.get("human_patches_passed") == 4 + passed = ( + all(item["passed"] for item in attempts) + and security_matches + and credential_present + and modes["control"] + and modes["treatment"] + and tool_inside + and fixture_passed + and encrypted + and not raw_path.exists() + ) + status = "passed" if passed else ("blocked" if not credential_present or not certificate else "failed") + if boundary_errors: + attempts.append({"id": "model_or_encryption_boundary", "passed": False, "reason": "; ".join(boundary_errors)}) + attempt_evidence = {item["id"]: item["passed"] for item in attempts} + record = { + "schema_version": "1.0.0", + "substrate_id": policy["substrate_id"], + "status": status, + "runner": { + "label": policy["runner_label"], + "image_os": os.environ.get("ImageOS"), + "image_version": os.environ.get("ImageVersion"), + }, + "container": { + "base_image": policy["base_image"], + "runtime_image_id": runtime.image_id(), + "security_configuration_matches": security_matches, + }, + "credential_boundary": { + "credential_present_in_controller": credential_present, + "credential_present_in_repair": not attempt_evidence.get("api_key", False), + "github_token_present_in_repair": not attempt_evidence.get("github_token", False), + }, + "attempts": attempts, + "fixture_revalidation": { + "buggy_failures": int(fixture.get("buggy_failures", 0)), + "human_patches_passed": int(fixture.get("human_patches_passed", 0)), + }, + "model_boundary": { + "model": policy["benchmark"]["model"], + "control_telemetry": modes["control"], + "treatment_telemetry": modes["treatment"], + "tool_call_executed_in_container": tool_inside, + "input_tokens": telemetry["input_tokens"], + "output_tokens": telemetry["output_tokens"], + "estimated_cost_usd": telemetry["estimated_cost_usd"], + }, + "raw_output": { + "encrypted": encrypted, + "plaintext_removed": not raw_path.exists(), + "format": policy["raw_output"]["format"], + }, + "benchmark_arms_executed": 0, + } + schema_validate(record, CANARY_SCHEMA, "canary result") + write_json(output, record) + if status != "passed": + raise ControllerError(f"hosted canary is {status}") + return record + + +def render_frozen_prompt(envelope): + spec = importlib.util.spec_from_file_location("frozen_arm_worker", BENCH / "arm_worker.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module.render_prompt(envelope) + + +def audit_frozen_bundle(manifest, pair, bundle, replicate, mode): + spec = importlib.util.spec_from_file_location("frozen_benchmark_runner", BENCH / "run_benchmark.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + module.run = lambda args, cwd=None, timeout=120, env=None, input_text=None: run( + args, + cwd=cwd, + input_text=input_text, + timeout=timeout, + check=False, + env=scrubbed_host_environment() if env is None else env, + ) + bundle = Path(bundle) + module.audit_bundle(manifest, pair, bundle) + if module.tree_sha256(bundle / "workspace") != pair["agent_fixture_sha256"]: + raise ControllerError("arm workspace differs from the frozen agent fixture") + with tempfile.TemporaryDirectory(prefix="aeg-expected-arm-") as raw: + expected_bundle = Path(raw) / "expected" + module.package_arm(manifest, pair, replicate, mode, expected_bundle) + if load_json(bundle / "arm.json") != load_json(expected_bundle / "arm.json"): + raise ControllerError("sanitized arm envelope differs from the frozen package") + return {"arm_id": f"{pair['pair_id']}--r{replicate:02d}--{mode}", "status": "passed"} + + +def patch_stats(diff_text): + added = deleted = 0 + for line in diff_text.splitlines(): + if line.startswith("+") and not line.startswith("+++"): + added += 1 + elif line.startswith("-") and not line.startswith("---"): + deleted += 1 + return added, deleted + + +def repeated_historical_paths(pair, evidence): + return [ + item["id"] + for item in pair["historical_failure_paths"] + if re.search(item["pattern"], evidence, re.IGNORECASE | re.DOTALL) + ] + + +def sanitized_assumption_metrics(pair, agent_result): + reported = { + item.get("assumption"): bool(item.get("checked")) + for item in agent_result.get("environment_assumptions_checked", []) + if isinstance(item, dict) and item.get("assumption") in pair["environment_assumptions"] + } + return [ + { + "assumption": assumption, + "checked": reported.get(assumption, False), + "evidence": ( + "agent structured result marked this preregistered assumption checked; raw evidence retained encrypted" + if reported.get(assumption, False) + else "agent did not mark this preregistered assumption checked" + ), + } + for assumption in pair["environment_assumptions"] + ] + + +def arm_coordinate(arm_id): + match = re.fullmatch(r"(s1-[a-z0-9-]+)--r(\d{2})--(control|aeg-assisted)", arm_id) + if not match: + raise ControllerError("arm id is invalid") + return match.group(1), int(match.group(2)), match.group(3) + + +def execute_arm(image, bundle, arm_id, sequence, sanitized_output, encrypted_output): + policy = load_json(POLICY) + manifest = load_json(MANIFEST) + plan = load_json(PLAN) + validate_configuration() + plan_ids = [item["arm_id"] for item in plan["arms"]] + if arm_id not in plan_ids: + raise ControllerError("arm is not in the frozen plan") + if sequence != plan_ids.index(arm_id) + 1: + raise ControllerError("arm sequence differs from the frozen plan") + pair_id, replicate, mode = arm_coordinate(arm_id) + pair = fixture_pair(manifest, pair_id) + bundle = Path(bundle) + audit_frozen_bundle(manifest, pair, bundle, replicate, mode) + envelope = load_json(bundle / "arm.json") + if envelope["arm_id"] != arm_id or envelope["mode"] != mode or envelope["replicate"] != replicate: + raise ControllerError("bundle coordinate differs from the frozen plan") + runtime = DockerRuntime(image, policy) + raw_dir = Path(tempfile.mkdtemp(prefix="aeg-arm-private-")) + raw_path = raw_dir / "raw.jsonl" + raw_path.write_text("", encoding="utf-8") + started = time.monotonic() + with tempfile.TemporaryDirectory(prefix="aeg-one-arm-") as raw: + repair_root = stage_root(bundle / "workspace", Path(raw) / "repair", bundle / "arm.json") + container = runtime.start(repair_root) + try: + bridge = ToolBridge( + runtime, + container, + policy, + envelope["budget"], + public_test_command=envelope["public_test_command"], + ) + client = ResponsesModelClient( + os.environ.get("OPENAI_API_KEY"), + envelope["model"], + policy, + bridge, + raw_path, + ) + final_text = client.run(render_frozen_prompt(envelope), final_schema=load_json(AGENT_RESULT_SCHEMA)) + agent_result = json.loads(final_text) + schema_validate(agent_result, AGENT_RESULT_SCHEMA, "agent result") + exported = runtime.exec( + container, + ["export-patch", "--max-bytes", str(policy["bridge"]["max_patch_bytes"])], + ) + finally: + runtime.remove(container) + patch_text = exported["patch"] + changed = exported["files_changed"] + with raw_path.open("a", encoding="utf-8") as handle: + handle.write(json.dumps({"kind": "repair_patch", "value": patch_text}, sort_keys=True) + "\n") + historical_evidence = patch_text + "\n" + raw_path.read_text(encoding="utf-8", errors="replace") + evaluator_root = Path(raw) / "evaluator" + evaluator_root.mkdir() + write_json(evaluator_root / "arm.json", minimal_arm(pair["hidden_test_command"])) + shutil.copytree(BENCH / "fixtures" / pair_id / "transfer" / "agent", evaluator_root / "task") + shutil.copyfile(BENCH / "fixtures" / pair_id / "transfer" / "evaluator" / "test_hidden.py", evaluator_root / "task" / "test_hidden.py") + evaluator = runtime.start(evaluator_root, prefix="aeg-evaluator") + try: + if patch_text: + runtime.exec(evaluator, ["apply-patch", "--max-bytes", str(policy["bridge"]["max_patch_bytes"])], input_text=patch_text) + evaluated = runtime.exec(evaluator, ["visible-test", "--timeout", "120"], timeout=125) + finally: + runtime.remove(evaluator) + added, deleted = patch_stats(patch_text) + telemetry = bridge.guard.telemetry() + if mode == "control": + experiences = [{"experience_id": None, "disposition": "abstained", "reason": "control mode has no AEG experience"}] + else: + disposition = agent_result["experience_disposition"] + experiences = [ + {"experience_id": envelope["experience_id"], "disposition": "retrieved", "reason": "frozen treatment payload delivered"}, + { + "experience_id": envelope["experience_id"], + "disposition": disposition, + "reason": f"agent structured result reported {disposition}; raw rationale retained encrypted", + }, + ] + result = { + "schema_version": "1.0.0", + "benchmark_id": envelope["benchmark_id"], + "family": "S1", + "pair_id": pair_id, + "replicate": replicate, + "mode": mode, + "evaluation_status": "evaluated", + "input_hashes": envelope["input_hashes"], + "budget": envelope["budget"], + "regression_free_success": evaluated["exit_code"] == 0, + "attempts": len(bridge.attempt_hashes), + "completed_commands": bridge.guard.commands, + "tests_run": bridge.tests + [{"command": pair["hidden_test_command"], "scope": "hidden", "passed": evaluated["exit_code"] == 0}], + "files_inspected": sorted(bridge.files_inspected), + "files_changed": changed, + "patch_size": {"added_lines": added, "deleted_lines": deleted, "files": len(changed)}, + "wall_time_ms": round((time.monotonic() - started) * 1000), + "tokens": {"input": telemetry["input_tokens"], "output": telemetry["output_tokens"], "unavailable_reason": None}, + "failed_historical_paths_repeated": repeated_historical_paths(pair, historical_evidence), + "environment_assumptions_checked": sanitized_assumption_metrics(pair, agent_result), + "experiences": experiences, + "negative_transfer": None, + "evaluator_findings": [ + f"hidden regression suite {'passed' if evaluated['exit_code'] == 0 else 'failed'}", + "repair container terminated before evaluator creation", + "evaluator received no human patch", + ], + "limitations": ["AEG Arm Execution Substrate v1", "negative transfer pending paired comparison"], + } + schema_validate(result, ARM_RESULT_SCHEMA, "sanitized arm result") + certificate = os.environ.get(policy["raw_output"]["public_certificate_environment"], "") + encrypted_raw_output(raw_path, encrypted_output, certificate) + write_json(sanitized_output, result) + return result + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest="action", required=True) + sub.add_parser("validate") + fixtures = sub.add_parser("revalidate-fixtures") + fixtures.add_argument("--image", required=True) + fixtures.add_argument("--output", required=True) + canary_parser = sub.add_parser("canary") + canary_parser.add_argument("--image", required=True) + canary_parser.add_argument("--fixture-record", required=True) + canary_parser.add_argument("--output", required=True) + canary_parser.add_argument("--encrypted-raw-output", required=True) + execute = sub.add_parser("execute-arm") + execute.add_argument("--image", required=True) + execute.add_argument("--bundle", required=True) + execute.add_argument("--arm-id", required=True) + execute.add_argument("--sequence", required=True, type=int) + execute.add_argument("--sanitized-output", required=True) + execute.add_argument("--encrypted-raw-output", required=True) + args = parser.parse_args() + if args.action == "validate": + result = validate_configuration() + elif args.action == "revalidate-fixtures": + result = revalidate_fixtures(args.image, args.output) + elif args.action == "canary": + result = canary(args.image, args.fixture_record, args.output, args.encrypted_raw_output) + else: + result = execute_arm(args.image, args.bundle, args.arm_id, args.sequence, args.sanitized_output, args.encrypted_raw_output) + print(json.dumps(result, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except (ControllerError, json.JSONDecodeError, subprocess.TimeoutExpired) as error: + print(f"arm substrate error: {error}", file=sys.stderr) + raise SystemExit(2) diff --git a/infrastructure/aeg-arm-execution-substrate/policy.json b/infrastructure/aeg-arm-execution-substrate/policy.json new file mode 100644 index 0000000..bb908cb --- /dev/null +++ b/infrastructure/aeg-arm-execution-substrate/policy.json @@ -0,0 +1,56 @@ +{ + "schema_version": "1.0.0", + "substrate_id": "aeg-arm-execution-substrate-v1", + "runner_label": "ubuntu-24.04", + "base_image": "python:3.12.11-slim-bookworm@sha256:519591d6871b7bc437060736b9f7456b8731f1499a57e22e6c285135ae657bf7", + "runtime_image_tag": "aeg-arm-runner:python3.12.11-slim-bookworm-v1", + "benchmark": { + "id": "situated-experience-benchmark-v1", + "manifest_sha256": "95ce8de8aca5580c8be95613b6058baecf2d473d9241831657e2a939577919c9", + "execution_plan_sha256": "6e6a3b75102d03d804cf0b8e1f51b3b1194fe5e1c39802b9d0cc64043bb9582a", + "model": "gpt-5.6-sol" + }, + "container": { + "network": "none", + "read_only_root": true, + "cap_drop": ["ALL"], + "no_new_privileges": true, + "pids_limit": 64, + "memory_bytes": 536870912, + "cpus": 1.0, + "tmpfs_bytes": 67108864, + "workspace_bytes": 33554432, + "workspace_storage": "tmpfs", + "host_bind_mounts": 0, + "allowed_process_environment": { + "HOME": "/nonexistent", + "LANG": "C.UTF-8", + "LC_ALL": "C.UTF-8", + "PATH": "/usr/local/bin:/usr/bin:/bin", + "PYTHONDONTWRITEBYTECODE": "1", + "PYTHONHASHSEED": "0" + } + }, + "bridge": { + "allowed_tools": ["inspect_file", "run_command", "apply_patch", "run_visible_tests"], + "max_file_read_bytes": 262144, + "max_patch_bytes": 262144, + "max_tool_output_bytes": 262144, + "max_model_turns": 160 + }, + "safety_ceiling": { + "max_total_tokens": 5000000, + "max_cost_usd": 50.0, + "max_output_tokens_per_response": 8192 + }, + "pricing_usd_per_million_tokens": { + "input": 5.0, + "cached_input": 0.5, + "output": 30.0 + }, + "raw_output": { + "format": "openssl-cms-der", + "cipher": "aes-256-cbc", + "public_certificate_environment": "AEG_RAW_OUTPUT_CERT_PEM" + } +} diff --git a/infrastructure/aeg-arm-execution-substrate/schemas/canary-result.schema.json b/infrastructure/aeg-arm-execution-substrate/schemas/canary-result.schema.json new file mode 100644 index 0000000..91dedf7 --- /dev/null +++ b/infrastructure/aeg-arm-execution-substrate/schemas/canary-result.schema.json @@ -0,0 +1,102 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://aeg.dev/schemas/arm-substrate-canary-v1.json", + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", + "substrate_id", + "status", + "runner", + "container", + "credential_boundary", + "attempts", + "fixture_revalidation", + "model_boundary", + "raw_output", + "benchmark_arms_executed" + ], + "properties": { + "schema_version": {"const": "1.0.0"}, + "substrate_id": {"const": "aeg-arm-execution-substrate-v1"}, + "status": {"enum": ["passed", "failed", "blocked"]}, + "runner": { + "type": "object", + "additionalProperties": false, + "required": ["label", "image_os", "image_version"], + "properties": { + "label": {"type": "string"}, + "image_os": {"type": ["string", "null"]}, + "image_version": {"type": ["string", "null"]} + } + }, + "container": { + "type": "object", + "additionalProperties": false, + "required": ["base_image", "runtime_image_id", "security_configuration_matches"], + "properties": { + "base_image": {"type": "string"}, + "runtime_image_id": {"type": ["string", "null"]}, + "security_configuration_matches": {"type": "boolean"} + } + }, + "credential_boundary": { + "type": "object", + "additionalProperties": false, + "required": ["credential_present_in_controller", "credential_present_in_repair", "github_token_present_in_repair"], + "properties": { + "credential_present_in_controller": {"type": "boolean"}, + "credential_present_in_repair": {"type": "boolean"}, + "github_token_present_in_repair": {"type": "boolean"} + } + }, + "attempts": { + "type": "array", + "minItems": 15, + "items": { + "type": "object", + "additionalProperties": false, + "required": ["id", "passed", "reason"], + "properties": { + "id": {"type": "string"}, + "passed": {"type": "boolean"}, + "reason": {"type": "string"} + } + } + }, + "fixture_revalidation": { + "type": "object", + "additionalProperties": false, + "required": ["buggy_failures", "human_patches_passed"], + "properties": { + "buggy_failures": {"type": "integer", "minimum": 0}, + "human_patches_passed": {"type": "integer", "minimum": 0} + } + }, + "model_boundary": { + "type": "object", + "additionalProperties": false, + "required": ["model", "control_telemetry", "treatment_telemetry", "tool_call_executed_in_container", "input_tokens", "output_tokens", "estimated_cost_usd"], + "properties": { + "model": {"const": "gpt-5.6-sol"}, + "control_telemetry": {"type": "boolean"}, + "treatment_telemetry": {"type": "boolean"}, + "tool_call_executed_in_container": {"type": "boolean"}, + "input_tokens": {"type": "integer", "minimum": 0}, + "output_tokens": {"type": "integer", "minimum": 0}, + "estimated_cost_usd": {"type": "number", "minimum": 0} + } + }, + "raw_output": { + "type": "object", + "additionalProperties": false, + "required": ["encrypted", "plaintext_removed", "format"], + "properties": { + "encrypted": {"type": "boolean"}, + "plaintext_removed": {"type": "boolean"}, + "format": {"const": "openssl-cms-der"} + } + }, + "benchmark_arms_executed": {"const": 0} + } +} diff --git a/infrastructure/aeg-arm-execution-substrate/tests/test_substrate.py b/infrastructure/aeg-arm-execution-substrate/tests/test_substrate.py new file mode 100644 index 0000000..f7e62e7 --- /dev/null +++ b/infrastructure/aeg-arm-execution-substrate/tests/test_substrate.py @@ -0,0 +1,378 @@ +import importlib.util +import json +import os +import shutil +import subprocess +import tempfile +import unittest +from unittest import mock +from pathlib import Path + + +HERE = Path(__file__).resolve().parent +SUBSTRATE = HERE.parent + + +def import_file(name, path): + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +controller = import_file("aeg_substrate_controller", SUBSTRATE / "controller.py") +worker = import_file("aeg_substrate_worker", SUBSTRATE / "container_worker.py") + + +class FakeRuntime: + def __init__(self, arm_root): + self.mount_roots = {"container": Path(arm_root)} + self.calls = [] + + def exec(self, container_id, arguments, input_text=None, timeout=120, check=True): + self.calls.append((container_id, arguments, input_text, timeout, check)) + if arguments[0] == "inspect": + return {"path": arguments[2], "content": "value\n"} + if arguments[0] == "run": + argv = json.loads(arguments[2]) + return {"argv": argv, "exit_code": 0, "stdout": "ok\n", "stderr": ""} + if arguments[0] == "visible-test": + return {"argv": ["python3", "-m", "unittest"], "exit_code": 0, "stdout": "ok\n", "stderr": ""} + if arguments[0] == "apply-patch": + return {"files_changed": ["module.py"]} + if arguments[0] == "snapshot": + return {"sha256": "a" * 64, "files": ["module.py"], "bytes": 32} + raise AssertionError(arguments) + + +class SubstrateConfigurationTests(unittest.TestCase): + def setUp(self): + self.policy = controller.load_json(controller.POLICY) + self.manifest = controller.load_json(controller.MANIFEST) + + def test_frozen_configuration_and_order_validate(self): + result = controller.validate_configuration() + self.assertEqual(result["manifest_sha256"], "95ce8de8aca5580c8be95613b6058baecf2d473d9241831657e2a939577919c9") + self.assertEqual(result["execution_plan_sha256"], "6e6a3b75102d03d804cf0b8e1f51b3b1194fe5e1c39802b9d0cc64043bb9582a") + self.assertEqual(result["arms"], 12) + + def test_docker_arguments_are_hardened_and_secret_free(self): + runtime = controller.DockerRuntime("test-image", self.policy) + with tempfile.TemporaryDirectory() as raw: + root = Path(raw) + (root / "task").mkdir() + (root / "arm.json").write_text("{}\n", encoding="utf-8") + args = runtime.security_args(root, "test-container") + joined = " ".join(args) + self.assertIn("--network none", joined) + self.assertNotIn("--pid", args) + self.assertNotIn("--pid host", joined) + self.assertIn("--read-only", args) + self.assertIn("--cap-drop ALL", joined) + self.assertIn("--security-opt no-new-privileges:true", joined) + self.assertIn("--pids-limit 64", joined) + self.assertIn("--memory 536870912", joined) + self.assertIn("--memory-swap 536870912", joined) + self.assertIn("--cpus 1", joined) + self.assertEqual(args.count("--mount"), 0) + self.assertEqual(args.count("--tmpfs"), 2) + self.assertIn("/workspace:rw,nosuid,nodev,size=33554432", joined) + self.assertNotIn("docker.sock", joined) + self.assertNotIn("OPENAI_API_KEY", joined) + self.assertNotIn("GITHUB_TOKEN", joined) + worker_command = runtime.worker_command("container-id", ["snapshot"]) + self.assertEqual(worker_command[:4], ["docker", "exec", "--interactive", "container-id"]) + + def test_host_tool_subprocess_environment_is_credential_scrubbed(self): + with mock.patch.dict(os.environ, { + "OPENAI_API_KEY": "fake-model-secret", + "GITHUB_TOKEN": "fake-github-secret", + "ACTIONS_ID_TOKEN_REQUEST_TOKEN": "fake-actions-secret", + "AEG_RAW_OUTPUT_CERT_PEM": "fake-public-cert", + }): + result = controller.run(["/usr/bin/env"]) + self.assertNotIn("fake-model-secret", result.stdout) + self.assertNotIn("fake-github-secret", result.stdout) + self.assertNotIn("fake-actions-secret", result.stdout) + self.assertNotIn("fake-public-cert", result.stdout) + + def test_bridge_exposes_exactly_four_strict_tools(self): + tools = controller.function_tools() + self.assertEqual( + [item["name"] for item in tools], + ["inspect_file", "run_command", "apply_patch", "run_visible_tests"], + ) + for item in tools: + self.assertTrue(item["strict"]) + self.assertFalse(item["parameters"]["additionalProperties"]) + + def test_worker_environment_matches_the_explicit_policy_allowlist(self): + self.assertEqual(worker.ALLOWED_ENV, self.policy["container"]["allowed_process_environment"]) + + def test_budget_guard_enforces_command_token_cost_and_disk_ceilings(self): + guard = controller.BudgetGuard( + self.policy, + {"wall_time_seconds": 900, "max_completed_commands": 1, "max_attempts": 3}, + ) + guard.add_command() + with self.assertRaisesRegex(controller.ControllerError, "command"): + guard.add_command() + with self.assertRaisesRegex(controller.ControllerError, "workspace"): + guard.check_workspace(self.policy["container"]["workspace_bytes"] + 1) + token_guard = controller.BudgetGuard( + self.policy, + {"wall_time_seconds": 900, "max_completed_commands": 40, "max_attempts": 3}, + ) + with self.assertRaisesRegex(controller.ControllerError, "token"): + token_guard.add_usage({"input_tokens": self.policy["safety_ceiling"]["max_total_tokens"] + 1}) + cost_guard = controller.BudgetGuard( + self.policy, + {"wall_time_seconds": 900, "max_completed_commands": 40, "max_attempts": 3}, + ) + with self.assertRaisesRegex(controller.ControllerError, "cost"): + cost_guard.add_usage({"output_tokens": 2_000_000}) + + +class ToolBridgeTests(unittest.TestCase): + def setUp(self): + self.raw = tempfile.TemporaryDirectory() + self.root = Path(self.raw.name) + controller.write_json(self.root / "arm.json", {"public_test_command": "python3 -m unittest -v test_public.py"}) + self.runtime = FakeRuntime(self.root) + self.bridge = controller.ToolBridge( + self.runtime, + "container", + controller.load_json(controller.POLICY), + {"wall_time_seconds": 900, "max_completed_commands": 40, "max_attempts": 3}, + public_test_command="python3 -m unittest -v test_public.py", + ) + + def tearDown(self): + self.raw.cleanup() + + def test_registered_test_via_run_command_is_measured(self): + result = self.bridge.call("run_command", {"command": "python3 -m unittest -v test_public.py"}) + self.assertEqual(result["exit_code"], 0) + self.assertEqual(self.bridge.guard.commands, 1) + self.assertEqual(self.bridge.tests, [{ + "command": "python3 -m unittest -v test_public.py", + "scope": "agent", + "passed": True, + }]) + + def test_unknown_tool_and_arguments_are_rejected(self): + with self.assertRaisesRegex(controller.ControllerError, "unknown"): + self.bridge.call("shell", {"command": "id"}) + with self.assertRaisesRegex(controller.ControllerError, "invalid"): + self.bridge.call("inspect_file", {"path": "module.py", "extra": True}) + + def test_attempts_count_unique_workspace_snapshots(self): + patch = "--- a/module.py\n+++ b/module.py\n@@ -1 +1 @@\n-old\n+new\n" + self.bridge.call("apply_patch", {"patch": patch}) + self.bridge.call("apply_patch", {"patch": patch}) + self.assertEqual(len(self.bridge.attempt_hashes), 1) + + +class WorkerBoundaryTests(unittest.TestCase): + def setUp(self): + self.raw = tempfile.TemporaryDirectory() + self.root = Path(self.raw.name) + self.task = self.root / "task" + self.task.mkdir() + self.arm = self.root / "arm.json" + self.arm.write_text(json.dumps({"public_test_command": "python3 -m unittest -v test_public.py"}), encoding="utf-8") + self.originals = worker.ROOT, worker.TASK, worker.ARM, worker.BASELINE + worker.ROOT, worker.TASK, worker.ARM, worker.BASELINE = self.root, self.task, self.arm, self.root / "baseline" + + def tearDown(self): + worker.ROOT, worker.TASK, worker.ARM, worker.BASELINE = self.originals + self.raw.cleanup() + + def test_patch_applies_only_to_existing_production_file(self): + module = self.task / "module.py" + module.write_text("old\n", encoding="utf-8") + patch = "diff --git a/module.py b/module.py\nindex 1111111..2222222 100644\n--- a/module.py\n+++ b/module.py\n@@ -1 +1 @@\n-old\n+new\n" + self.assertEqual(worker.parse_and_apply_patch(patch, 4096), ["module.py"]) + self.assertEqual(module.read_text(encoding="utf-8"), "new\n") + + def test_patch_rejects_tests_traversal_symlink_and_duplicate_target(self): + (self.task / "test_public.py").write_text("old\n", encoding="utf-8") + protected = "--- a/test_public.py\n+++ b/test_public.py\n@@ -1 +1 @@\n-old\n+new\n" + with self.assertRaisesRegex(worker.WorkerError, "protected"): + worker.parse_and_apply_patch(protected, 4096) + traversal = "--- a/../outside.py\n+++ b/../outside.py\n@@ -1 +1 @@\n-old\n+new\n" + with self.assertRaisesRegex(worker.WorkerError, "outside"): + worker.parse_and_apply_patch(traversal, 4096) + outside = self.root / "outside.py" + outside.write_text("old\n", encoding="utf-8") + (self.task / "link.py").symlink_to(outside) + symlink = "--- a/link.py\n+++ b/link.py\n@@ -1 +1 @@\n-old\n+new\n" + with self.assertRaisesRegex(worker.WorkerError, "escapes|symlink"): + worker.parse_and_apply_patch(symlink, 4096) + module = self.task / "module.py" + module.write_text("old\n", encoding="utf-8") + duplicate = ( + "--- a/module.py\n+++ b/module.py\n@@ -1 +1 @@\n-old\n+one\n" + "--- a/module.py\n+++ b/module.py\n@@ -1 +1 @@\n-old\n+two\n" + ) + with self.assertRaisesRegex(worker.WorkerError, "only once"): + worker.parse_and_apply_patch(duplicate, 4096) + + def test_all_four_registered_human_patches_apply_to_frozen_seeds(self): + manifest = controller.load_json(controller.MANIFEST) + for pair in manifest["pairs"]: + for stage in ("source", "transfer"): + fixture = controller.BENCH / "fixtures" / pair["pair_id"] / stage + seed = "buggy" if stage == "source" else "agent" + case = self.root / f"{pair['pair_id']}-{stage}" + shutil.copytree(fixture / seed, case) + previous = worker.TASK + worker.TASK = case + try: + changed = worker.parse_and_apply_patch( + (fixture / "evaluator" / "human.patch").read_text(encoding="utf-8"), + 262144, + ) + finally: + worker.TASK = previous + self.assertTrue(changed, f"{pair['pair_id']} {stage}") + + def test_command_allowlist_and_environment(self): + result = worker.run_registered(["python3", "--version"], 10) + self.assertEqual(result["exit_code"], 0) + with self.assertRaisesRegex(worker.WorkerError, "allowlisted"): + worker.run_registered(["sh", "-c", "env"], 10) + + def test_baseline_export_includes_only_modified_production_file(self): + module = self.task / "module.py" + module.write_text("old\n", encoding="utf-8") + (self.task / "ISSUE.md").write_text("protected\n", encoding="utf-8") + worker.create_baseline() + module.write_text("new\n", encoding="utf-8") + result = worker.export_workspace_patch(4096) + self.assertEqual(result["files_changed"], ["module.py"]) + self.assertIn("--- a/module.py", result["patch"]) + self.assertNotIn("ISSUE.md", result["patch"]) + (self.task / "new.py").write_text("new\n", encoding="utf-8") + with self.assertRaisesRegex(worker.WorkerError, "creation"): + worker.export_workspace_patch(4096) + + def test_sanitized_bundle_stream_imports_into_empty_tmpfs(self): + source = self.root / "source" + source.mkdir() + (source / "arm.json").write_text('{"public_test_command":"python3 --version"}\n', encoding="utf-8") + source_task = source / "task" + source_task.mkdir() + (source_task / "module.py").write_text("VALUE = 1\n", encoding="utf-8") + payload = controller.encoded_sanitized_bundle(source, 4096) + imported_root = self.root / "imported" + imported_root.mkdir() + current = worker.ROOT, worker.TASK, worker.ARM + worker.ROOT, worker.TASK, worker.ARM = imported_root, imported_root / "task", imported_root / "arm.json" + try: + result = worker.import_bundle(payload, 4096) + finally: + worker.ROOT, worker.TASK, worker.ARM = current + self.assertEqual(result["bytes"], 54) + self.assertEqual((imported_root / "task" / "module.py").read_text(encoding="utf-8"), "VALUE = 1\n") + + +class ModelAndEncryptionTests(unittest.TestCase): + def setUp(self): + self.policy = controller.load_json(controller.POLICY) + self.raw = tempfile.TemporaryDirectory() + self.root = Path(self.raw.name) + controller.write_json(self.root / "arm.json", {"public_test_command": "python3 --version"}) + self.runtime = FakeRuntime(self.root) + self.bridge = controller.ToolBridge( + self.runtime, + "container", + self.policy, + {"wall_time_seconds": 900, "max_completed_commands": 40, "max_attempts": 3}, + public_test_command="python3 --version", + ) + + def tearDown(self): + self.raw.cleanup() + + def test_model_client_runs_tools_on_bridge_and_retains_telemetry(self): + class FakeClient(controller.ResponsesModelClient): + def __init__(inner, *args, **kwargs): + super().__init__(*args, **kwargs) + inner.payloads = [] + + def request(inner, payload): + inner.payloads.append(payload) + if len(inner.payloads) == 1: + return { + "output": [{ + "type": "function_call", + "name": "inspect_file", + "call_id": "call-1", + "arguments": json.dumps({"path": "CANARY.txt"}), + }], + "usage": {"input_tokens": 10, "output_tokens": 5}, + } + return { + "output": [{"type": "message", "content": [{"type": "output_text", "text": "complete"}]}], + "usage": {"input_tokens": 8, "output_tokens": 3}, + } + + raw_path = self.root / "raw.jsonl" + client = FakeClient("host-secret", "gpt-5.6-sol", self.policy, self.bridge, raw_path) + result = client.run("canary", forced_first_tool="inspect_file") + self.assertEqual(result, "complete") + self.assertEqual(self.bridge.files_inspected, {"CANARY.txt"}) + self.assertEqual(self.bridge.guard.telemetry()["input_tokens"], 18) + self.assertTrue(all(payload["store"] is False for payload in client.payloads)) + self.assertNotIn("host-secret", json.dumps(client.payloads)) + + def test_missing_certificate_deletes_plaintext(self): + raw_path = self.root / "private.jsonl" + raw_path.write_text("secret\n", encoding="utf-8") + with self.assertRaisesRegex(controller.ControllerError, "certificate"): + controller.encrypted_raw_output(raw_path, self.root / "raw.p7m", "") + self.assertFalse(raw_path.exists()) + + @unittest.skipUnless(shutil.which("openssl"), "openssl is required") + def test_encrypted_raw_output_round_trip_and_plaintext_removal(self): + cert = self.root / "cert.pem" + key = self.root / "key.pem" + subprocess.run([ + "openssl", "req", "-x509", "-newkey", "rsa:2048", "-nodes", + "-keyout", str(key), "-out", str(cert), "-subj", "/CN=aeg-test", "-days", "1", + ], check=True, capture_output=True) + raw_path = self.root / "private.jsonl" + encrypted = self.root / "private.p7m" + decrypted = self.root / "decrypted.jsonl" + raw_path.write_text("private model output\n", encoding="utf-8") + self.assertTrue(controller.encrypted_raw_output(raw_path, encrypted, cert.read_text(encoding="utf-8"))) + self.assertFalse(raw_path.exists()) + subprocess.run([ + "openssl", "cms", "-decrypt", "-binary", "-inform", "DER", + "-in", str(encrypted), "-recip", str(cert), "-inkey", str(key), "-out", str(decrypted), + ], check=True, capture_output=True) + self.assertEqual(decrypted.read_text(encoding="utf-8"), "private model output\n") + + def test_historical_path_detection_uses_registered_patterns(self): + pair = controller.fixture_pair(controller.load_json(controller.MANIFEST), "s1-02-fastapi-pydantic") + repeated = controller.repeated_historical_paths(pair, "field.type_ is list") + self.assertEqual(repeated, ["special_case_list_only"]) + + def test_assumption_sanitizer_retains_no_agent_evidence(self): + pair = controller.fixture_pair(controller.load_json(controller.MANIFEST), "s1-02-fastapi-pydantic") + agent_result = { + "environment_assumptions_checked": [{ + "assumption": pair["environment_assumptions"][0], + "checked": True, + "evidence": "PRIVATE RAW MODEL EVIDENCE", + }] + } + sanitized = controller.sanitized_assumption_metrics(pair, agent_result) + self.assertTrue(sanitized[0]["checked"]) + self.assertNotIn("PRIVATE RAW MODEL EVIDENCE", json.dumps(sanitized)) + self.assertEqual(len(sanitized), len(pair["environment_assumptions"])) + + +if __name__ == "__main__": + unittest.main() diff --git a/integrations/vscode/CHANGELOG.md b/integrations/vscode/CHANGELOG.md index a80e90a..e9e280c 100644 --- a/integrations/vscode/CHANGELOG.md +++ b/integrations/vscode/CHANGELOG.md @@ -1,5 +1,22 @@ # Changelog +## 0.1.6 + +- Make **AEG: Start with Verified Experience** the single dominant entry point + while preserving the previous command ID as a compatibility alias. +- Show the two-record/two-family coverage boundary and render abstention as a + correct, explained outcome with no fallback injection. +- Enforce evidence inspection, clipboard handoff, objective validation, and + query/experience-linked local feedback in order. +- Add exact normal-Chat paste instructions without using undocumented APIs. +- Move Playwright, Repair Lab, skill discovery, the bundled challenge, and + legacy commands into an Advanced sidebar section. +- Add a five-step founder walkthrough, UX state-transition tests, and + non-executing three-arm protocol/result schema tests. +- Fix the closed-window command-line VSIX install gap with deferred startup + activation, a versioned global first-run marker, retryable failure states, + manual reopen, and a persistent **AEG: Start here** status-bar fallback. + ## 0.1.5 - Rank verified experiences against top-level lessons, subtask descriptions, diff --git a/integrations/vscode/README.md b/integrations/vscode/README.md index 66349df..1d4f63d 100644 --- a/integrations/vscode/README.md +++ b/integrations/vscode/README.md @@ -1,160 +1,115 @@ # Agent Experience Graph for VS Code -Retrieve verified debugging experience before your coding agent starts from -scratch. - -AEG v0.1.5 is a local-first developer preview: - -**Task or error → Explainable verified match → Guarded recovery capsule → Local rating** - -## What is new in v0.1.5 - -- Verified-experience retrieval now considers lessons and subtask evidence - without allowing long records to accumulate an unfair score advantage. -- Match explanations show the query terms that actually overlap, and nonzero - near-matches are disclosed when AEG abstains below its retrieval threshold. - -## Marketplace icon restored in v0.1.4 - -- Restore the original AEG Marketplace icon and package it explicitly. - -## Verified-experience workflow introduced in v0.1.3 - -- **AEG: Try a Verified Experience** searches the bundled, validated public - library. Candidate and malformed records are not eligible. -- Each card shows why it matched, the validated outcome, reusable lessons, - recommended use cases, constraints, limitations, and public provenance. -- **Copy capsule** produces concise context for a coding agent with an explicit - instruction to inspect the local code and run focused and regression tests. -- Helpful, partially helpful, irrelevant, and harmful ratings stay in - `.aeg/verified-experience-feedback.json`; task text and ratings are not - uploaded. -- **AEG: Open Verified Experience Challenge** opens a bundled synthetic - transfer task so a new user can see the full product loop immediately. -- **AEG: Run Public Repair Lab** launches isolated repairs of a real, - MIT-licensed FastAPI nested response-model bug by default. -- The baseline and assisted arms receive identical issue text, code, and tests; - only the assisted arm receives a compact retrieved recovery capsule. -- Repeated paired trials alternate execution order. Corrected telemetry captures - duration, completed commands, actual test runs, token usage, changed files, and - patches under `.aeg/repair-lab/`. -- The runner uses `codex exec --ephemeral --sandbox workspace-write`; it never - pushes code or contacts the upstream project. -- Verdicts require at least three trials and remain specific to the selected task. - -Run it from the AEG sidebar or command palette. The local `codex` executable -must be available on `PATH`. - -## First verified-experience retrieval - -1. Open a project in VS Code. -2. Select an error or describe a task with **AEG: Try a Verified Experience**. -3. Choose a match and inspect **Why this matched** and its limitations. -4. Copy the compact capsule into the coding-agent session before it begins the - repair. -5. Validate the result locally, then record whether retrieval was helpful. - -For an immediate demo, run **AEG: Open Verified Experience Challenge**. This is -a synthetic, non-identical transfer fixture. Its prior A/B pair produced the -same successful patch in both arms; retrieval changed neither repair path nor -outcome and increased token usage and wall time. It demonstrates the workflow, -not an AEG benefit claim. - -## Playwright diagnosis (from v0.1.1) - -- A dedicated **AEG Playwright** sidebar and status-bar entry point. -- Failure input from selected text, the latest Playwright artifact, the active file, a copied error, or a short description. -- Ten bundled Playwright recovery playbooks: - - timeouts - - unstable selectors - - authentication and session state - - network and API mocking - - flaky tests - - browser-specific failures - - test-data isolation - - CI-only failures - - trace and artifact diagnosis - - accessibility failures -- Local experience receipts using the minimum AEG structure: - - Intent - - Context - - Steps - - Skills - - Artifacts - - Failures - - Recovery - - Outcome - - Cost -- Explicit resolved/unresolved verification after a recovery attempt. -- Automatic detection of new text-based artifacts under `test-results`. - -## First diagnosis - -1. Open a project in VS Code. -2. Run a Playwright test and copy its error, or select an error/stack trace in the editor. -3. Click **AEG Playwright** in the Activity Bar or status bar. -4. Choose **Diagnose Playwright failure**. -5. Select a recommended playbook and try its recovery steps. -6. Re-run the test and mark the outcome **Test passed** or **Still failing**. - -AEG stores the receipt under: +AEG v0.1.6 is a local-first product-proof release with one primary workflow: -```text -.aeg/experiences/ +**Task or error → verified match or explicit abstention → evidence and limitations → guarded capsule handoff → observed validation → local feedback** + +AEG retrieves guidance. It does not automatically solve, send, or run the task. + +## Install a local build + +```bash +cd integrations/vscode +npm ci +npm test +npm run package +code --install-extension agent-experience-graph-0.1.6.vsix --force ``` -Use **AEG: Show Playwright Experiences** to inspect prior receipts. +Open any test workspace in a fresh VS Code window. On first activation in a +normal workspace, the five-step walkthrough opens once and stores a versioned +profile marker. Later windows do not force it open. It remains available from +the AEG sidebar as **Guided walkthrough** and through **AEG: Open Founder Proof +Walkthrough**. If automatic opening is skipped or fails, the status bar still +shows **AEG: Start here**. -## Privacy +The initial founder run exposed a discovery failure, and the onboarding fix was +then re-tested successfully in a clean profile on 2026-08-14 using VSIX +SHA-256 +`18ef493b9290e28832e54527d7fb92624387a17d749ec228b60087c3b6917224`. +The founder usability/discoverability gate passed: first-workspace auto-open, +all five steps, local feedback creation, second-workspace suppression, and +manual reopen were confirmed. The product-proof experiment remains prepared, +not frozen, with 0/3 arms executed. + +## Golden path + +1. Select error text or run **AEG: Start with Verified Experience** and enter a task. +2. AEG searches the bundled verified-only library locally. +3. If a result clears the fixed threshold, select it and inspect the exact matching phrases, weighted score, verified source outcome, provenance, constraints, and limitations. +4. Select **Copy capsule**. Open VS Code Chat from the Chat menu (macOS: Control+Command+I; Windows/Linux: Ctrl+Alt+I), paste into the chat input with the original task, and press Enter. For another coding agent, paste into its normal task or prompt input before it starts. +5. Run focused and regression checks. Record **Checks passed**, **Partially passed**, **Still failing**, or **Did not apply**. +6. Rate the selected experience **Helpful**, **Partially helpful**, **Irrelevant**, or **Harmful**. + +The panel keeps the original query and selected experience ID visible through validation and rating. It will not enable validation before handoff or rating before validation. + +## Honest coverage and abstention + +The public library contains exactly two verified records in two narrow task families: -Version 0.1.5 does not upload code, task descriptions, recovery capsules, logs, -artifacts, ratings, or experience receipts. +- agent evaluation and telemetry integrity; +- delegation and API contract repair. -- Receipts are local by default. -- Common authorization headers, passwords, tokens, API keys, and credential-bearing URLs are redacted from captured failure signatures. -- Artifact paths are recorded, but raw artifact contents are not written into receipts. -- Sharing is intentionally excluded until AEG has an explicit preview, consent, and redaction flow. +Retrieval is deterministic lexical ranking with a fixed 0.0500 threshold. **No relevant verified experience** is a correct outcome: AEG explains the best score or zero-score result, shows current coverage, and injects no candidate or generic fallback guidance. -Review a receipt before committing `.aeg/` to source control. Add `.aeg/` to `.gitignore` if the repository should not retain local experience data. +Verified means the recorded source outcome was objectively checked. It does not mean AEG retrieval improved correctness, success, speed, cost, or generalization. -## Existing skill commands +## Handoff API decision -The v0.1.0 local-skill workflow remains available: +VS Code documents `vscode.editorChat.start` for editor chat, but no stable extension API to open and prefill the normal Chat view. v0.1.6 therefore uses the supported clipboard API plus explicit paste-and-run instructions. It does not call an undocumented or private workbench command and never submits the capsule automatically. + +## Local feedback + +Feedback is appended to: + +```text +.aeg/verified-experience-feedback.json +``` + +Each row links a local proof-loop session, redacted query summary, selected experience ID and task, retrieval score, observed validation outcome, and rating. Review or ignore `.aeg/` before committing it. + +## Advanced capabilities + +The sidebar keeps prior capabilities under a collapsed **Advanced** section, and every existing command ID remains registered for backward compatibility: + +- bundled synthetic transfer challenge; +- Playwright artifact diagnosis, local receipts, and outcome marking; +- Public Repair Lab; +- workspace skill discovery, recommendation, rating, and local metrics; +- legacy getting-started content. + +The synthetic challenge demonstrates the interaction flow only. Its prior pair produced the same successful patch and repair path in both arms while assisted tokens and wall time were higher. The legacy Repair Lab and Playwright tools are not part of the default v0.1.6 path. + +## Privacy -- `AEG: Discover Workspace Skills` -- `AEG: Recommend Skill for Current Task` -- `AEG: Rate a Skill` -- `AEG: Show Skill Metrics` +AEG v0.1.6 does not upload code, task descriptions, prompts, recovery capsules, logs, artifacts, ratings, receipts, or private data. -These commands scan local `SKILL.md` and `capability.json` files. Skill metrics remain local in `.aeg/skill-metrics.json`. +- Search, ranking, clipboard handoff, and feedback are local. +- Common credentials are redacted from captured Playwright failure signatures. +- Raw Playwright artifact content is not written into receipts. +- There is no telemetry or sharing path in the verified-experience workflow. ## Development ```bash -npm install +npm ci npm test +npm run compile npm run package ``` -Open the extension directory in VS Code and press `F5` to launch an Extension Development Host. +`npm test` compiles TypeScript, runs extension retrieval, proof-loop, and +first-run state-transition tests, validates the product-proof protocol/result +schemas, and verifies walkthrough assets. Packaging synchronizes the unchanged +two-record verified library into the VSIX. ## Current limitations -- The verified public library contains only two records and supports no claim - of general coverage. -- Verified means the recorded outcome was objectively checked; it does not mean - AEG retrieval caused an improvement. -- Retrieval is deterministic lexical ranking, not embedding-based semantic - search. No match above the threshold means AEG abstains. -- The bundled transfer challenge is synthetic and is not cross-project - validation. Its prior controlled pair found no correctness or efficiency - benefit. -- Playbook ranking is deterministic keyword/signature matching, not semantic retrieval. -- AEG cannot read arbitrary integrated-terminal output; use a selection, clipboard, file, or Playwright artifact. -- Test outcome verification is user-confirmed in this release. -- Token counts are estimates based on captured text length. - -These constraints keep the first data loop understandable and auditable while -AEG recruits 5–10 seed users to test whether verified experience is useful on -their real debugging tasks. Please report a concrete retrieval outcome through -the repository issue tracker. +- The verified library has only two records and does not provide broad coverage. +- Retrieval is lexical, not embedding-based semantic search. +- The user confirms objective validation outcomes; AEG does not independently run or observe the checks. +- The normal Chat handoff is manual because no documented stable prefilled-Chat API is used. +- Automatic onboarding is scoped to the first activation with a workspace; an + empty window keeps the marker unset and exposes the status-bar fallback. +- The bundled challenge demonstrates discoverability and interaction, not performance benefit. +- Prior transfer evidence is neutral or negative and supports no claim of improved success, speed, cost, PMF, adoption, or generalization. diff --git a/integrations/vscode/package-lock.json b/integrations/vscode/package-lock.json index 8e65cf7..cff1f4a 100644 --- a/integrations/vscode/package-lock.json +++ b/integrations/vscode/package-lock.json @@ -1,17 +1,18 @@ { "name": "agent-experience-graph", - "version": "0.1.5", + "version": "0.1.6", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "agent-experience-graph", - "version": "0.1.5", + "version": "0.1.6", "license": "MIT-0", "devDependencies": { "@types/node": "^20.14.0", "@types/vscode": "^1.90.0", "@vscode/vsce": "^3.6.2", + "ajv": "^8.20.0", "typescript": "^5.5.0" }, "engines": { diff --git a/integrations/vscode/package.json b/integrations/vscode/package.json index 33cea21..849b39e 100644 --- a/integrations/vscode/package.json +++ b/integrations/vscode/package.json @@ -1,8 +1,8 @@ { "name": "agent-experience-graph", "displayName": "Agent Experience Graph", - "description": "Retrieve verified debugging experience before your coding agent starts from scratch, with local-only feedback and auditable evidence.", - "version": "0.1.5", + "description": "Search a small verified-experience library, inspect evidence and limitations, hand off guarded guidance, validate the outcome, and save local feedback.", + "version": "0.1.6", "publisher": "AgentExperienceGraph", "icon": "images/icon.png", "license": "MIT-0", @@ -24,20 +24,25 @@ "Other" ], "keywords": [ - "playwright", - "testing", "debugging", "agent", - "skills", - "experience" + "experience", + "verified", + "playwright", + "testing" ], "activationEvents": [ + "onStartupFinished", "onView:aeg.playwright", + "onCommand:aeg.startWithVerifiedExperience", "onCommand:aeg.tryVerifiedExperience", + "onCommand:aeg.showVerifiedCoverage", + "onCommand:aeg.openFounderWalkthrough", "onCommand:aeg.openVerifiedExperienceDemo", "onCommand:aeg.diagnosePlaywrightFailure", "onCommand:aeg.verifyLatestExperience", "onCommand:aeg.showExperiences", + "onCommand:aeg.openGettingStarted", "onCommand:aeg.runPublicRepairLab", "onCommand:aeg.discoverSkills", "onCommand:aeg.recommendSkill", @@ -48,60 +53,84 @@ "contributes": { "commands": [ { - "command": "aeg.tryVerifiedExperience", - "title": "AEG: Try a Verified Experience", + "command": "aeg.startWithVerifiedExperience", + "title": "AEG: Start with Verified Experience", "icon": "$(library)" }, + { + "command": "aeg.tryVerifiedExperience", + "title": "AEG: Try a Verified Experience (Compatibility Alias)" + }, + { + "command": "aeg.showVerifiedCoverage", + "title": "AEG: Show Verified Library Coverage", + "icon": "$(book)" + }, + { + "command": "aeg.openFounderWalkthrough", + "title": "AEG: Open Founder Proof Walkthrough", + "icon": "$(map)" + }, { "command": "aeg.openVerifiedExperienceDemo", - "title": "AEG: Open Verified Experience Challenge", + "title": "AEG (Advanced): Open Bundled Transfer Challenge", "icon": "$(lightbulb-autofix)" }, { "command": "aeg.diagnosePlaywrightFailure", - "title": "AEG: Diagnose Playwright Failure", + "title": "AEG (Advanced): Diagnose Playwright Failure", "icon": "$(debug-alt)" }, { "command": "aeg.verifyLatestExperience", - "title": "AEG: Mark Latest Diagnosis Outcome", + "title": "AEG (Advanced): Mark Latest Playwright Outcome", "icon": "$(pass)" }, { "command": "aeg.showExperiences", - "title": "AEG: Show Playwright Experiences", + "title": "AEG (Advanced): Show Playwright Experiences", "icon": "$(history)" }, { "command": "aeg.openGettingStarted", - "title": "AEG: Playwright Getting Started" + "title": "AEG (Advanced): Open Legacy Getting Started" }, { "command": "aeg.runPublicRepairLab", - "title": "AEG: Run Public Repair Lab", + "title": "AEG (Advanced): Run Public Repair Lab", "icon": "$(beaker)" }, { "command": "aeg.discoverSkills", - "title": "AEG: Discover Workspace Skills" + "title": "AEG (Advanced): Discover Workspace Skills" }, { "command": "aeg.recommendSkill", - "title": "AEG: Recommend Skill for Current Task" + "title": "AEG (Advanced): Recommend Skill for Current Task" }, { "command": "aeg.rateSkill", - "title": "AEG: Rate a Skill" + "title": "AEG (Advanced): Rate a Skill" }, { "command": "aeg.showSkillMetrics", - "title": "AEG: Show Skill Metrics" + "title": "AEG (Advanced): Show Skill Metrics" } ], "menus": { "commandPalette": [ { - "command": "aeg.tryVerifiedExperience" + "command": "aeg.startWithVerifiedExperience" + }, + { + "command": "aeg.tryVerifiedExperience", + "when": "false" + }, + { + "command": "aeg.showVerifiedCoverage" + }, + { + "command": "aeg.openFounderWalkthrough" }, { "command": "aeg.openVerifiedExperienceDemo" @@ -136,41 +165,21 @@ ], "editor/context": [ { - "command": "aeg.tryVerifiedExperience", + "command": "aeg.startWithVerifiedExperience", "when": "editorHasSelection", "group": "navigation@1" - }, - { - "command": "aeg.diagnosePlaywrightFailure", - "when": "editorHasSelection", - "group": "navigation@2" - }, - { - "command": "aeg.recommendSkill", - "when": "editorHasSelection", - "group": "navigation@3" } ], "view/title": [ { - "command": "aeg.tryVerifiedExperience", + "command": "aeg.startWithVerifiedExperience", "when": "view == aeg.playwright", "group": "navigation@1" }, { - "command": "aeg.diagnosePlaywrightFailure", + "command": "aeg.showVerifiedCoverage", "when": "view == aeg.playwright", "group": "navigation@2" - }, - { - "command": "aeg.showExperiences", - "when": "view == aeg.playwright", - "group": "navigation@3" - }, - { - "command": "aeg.runPublicRepairLab", - "when": "view == aeg.playwright", - "group": "navigation@4" } ] }, @@ -187,14 +196,79 @@ "aeg": [ { "id": "aeg.playwright", - "name": "Verified Experience & Playwright" + "name": "Verified Experience" } ] }, "viewsWelcome": [ { "view": "aeg.playwright", - "contents": "Retrieve verified debugging experience before your coding agent starts from scratch.\n[Try a verified experience](command:aeg.tryVerifiedExperience)\n[Open the bundled challenge](command:aeg.openVerifiedExperienceDemo)\n[Diagnose a Playwright failure](command:aeg.diagnosePlaywrightFailure)\n[Run public repair lab](command:aeg.runPublicRepairLab)\n[Open getting started](command:aeg.openGettingStarted)" + "contents": "Search two bundled verified records locally. AEG may correctly abstain.\n[Start with Verified Experience](command:aeg.startWithVerifiedExperience)\n[View coverage](command:aeg.showVerifiedCoverage)\n[Open guided walkthrough](command:aeg.openFounderWalkthrough)" + } + ], + "walkthroughs": [ + { + "id": "aegFounderProofLoop", + "title": "AEG verified-experience proof loop", + "description": "Complete one honest path from task or error to local outcome feedback.", + "steps": [ + { + "id": "start", + "title": "1. Start with a task or error", + "description": "Select an error first or enter a short task. Search stays local.\n[Start with Verified Experience](command:aeg.startWithVerifiedExperience)\n[Use the bundled guided task](command:aeg.openVerifiedExperienceDemo)", + "media": { + "markdown": "walkthrough/01-start.md" + }, + "completionEvents": [ + "onCommand:aeg.startWithVerifiedExperience", + "onCommand:aeg.openVerifiedExperienceDemo" + ] + }, + { + "id": "inspect", + "title": "2. Inspect the match or abstention", + "description": "Read the matching phrases, provenance, constraints, and limitations. A clear abstention is a correct outcome.", + "media": { + "markdown": "walkthrough/02-inspect.md" + }, + "completionEvents": [ + "onContext:aeg.proofLoopInspected || aeg.proofLoopAbstained" + ] + }, + { + "id": "handoff", + "title": "3. Copy and paste the guarded capsule", + "description": "Copy only an above-threshold capsule, then paste it into the normal chat input. AEG does not send or run it.", + "media": { + "markdown": "walkthrough/03-handoff.md" + }, + "completionEvents": [ + "onContext:aeg.proofLoopCopied" + ] + }, + { + "id": "validate", + "title": "4. Validate the outcome", + "description": "Run focused and regression checks, then record the observed result in the proof-loop panel.", + "media": { + "markdown": "walkthrough/04-validate.md" + }, + "completionEvents": [ + "onContext:aeg.proofLoopValidated" + ] + }, + { + "id": "rate", + "title": "5. Save local feedback", + "description": "Rate the selected experience as helpful, partially helpful, irrelevant, or harmful. The record stays in this workspace.", + "media": { + "markdown": "walkthrough/05-rate.md" + }, + "completionEvents": [ + "onContext:aeg.proofLoopRated" + ] + } + ] } ], "configuration": { @@ -221,7 +295,7 @@ "aeg.verifiedExperienceFeedbackFile": { "type": "string", "default": ".aeg/verified-experience-feedback.json", - "description": "Workspace-relative file for local usefulness ratings of retrieved verified experiences." + "description": "Workspace-relative file for local validation outcomes and usefulness ratings of retrieved verified experiences." }, "aeg.playwrightArtifactGlobs": { "type": "array", @@ -247,13 +321,14 @@ "sync:repair-lab": "node scripts/sync-repair-lab.js", "compile": "npm run sync:repair-lab && tsc -p ./", "watch": "tsc -watch -p ./", - "test": "npm run compile && node --test test/core.test.js", + "test": "npm run compile && node --test test/*.test.js", "package": "npx @vscode/vsce package" }, "devDependencies": { "@types/node": "^20.14.0", "@types/vscode": "^1.90.0", "@vscode/vsce": "^3.6.2", + "ajv": "^8.20.0", "typescript": "^5.5.0" } } diff --git a/integrations/vscode/src/extension.ts b/integrations/vscode/src/extension.ts index 8bc4e2e..bde2bed 100644 --- a/integrations/vscode/src/extension.ts +++ b/integrations/vscode/src/extension.ts @@ -19,8 +19,24 @@ import { describeBelowThresholdMatch, generateRecoveryCapsule, loadVerifiedExperienceLibrary, - rankVerifiedExperiences + rankVerifiedExperiences, + summarizeVerifiedLibraryCoverage } from './verifiedExperience'; +import { + ProofLoopSession, + ValidationOutcome, + beginProofLoop, + proofLoopStep, + transitionProofLoop +} from './proofLoop'; +import { + FOUNDER_FIRST_RUN_MARKER_KEY, + FOUNDER_WALKTHROUGH_ID, + PRIMARY_ENTRY_STATUS_TEXT, + FounderFirstRunStorage, + openFounderWalkthroughOnFirstRun, + reopenFounderWalkthrough +} from './firstRun'; const VERIFIED_EXPERIENCE_DEMO = 'Keepalive control fails after active stream ownership moved behind a protocol object; repair the public wrapper so it delegates through the protocol without using its stale socket field.'; @@ -55,9 +71,13 @@ class AegTreeItem extends vscode.TreeItem { label: string, description: string, icon: string, - command?: vscode.Command + command?: vscode.Command, + readonly children: AegTreeItem[] = [] ) { - super(label, vscode.TreeItemCollapsibleState.None); + super( + label, + children.length ? vscode.TreeItemCollapsibleState.Collapsed : vscode.TreeItemCollapsibleState.None + ); this.description = description; this.iconPath = new vscode.ThemeIcon(icon); this.command = command; @@ -68,6 +88,8 @@ class PlaywrightViewProvider implements vscode.TreeDataProvider { private readonly changeEmitter = new vscode.EventEmitter(); readonly onDidChangeTreeData = this.changeEmitter.event; + constructor(private readonly extensionUri: vscode.Uri) {} + refresh(): void { this.changeEmitter.fire(undefined); } @@ -76,18 +98,17 @@ class PlaywrightViewProvider implements vscode.TreeDataProvider { return element; } - async getChildren(): Promise { - const count = await countExperienceReceipts(); - return [ + async getChildren(element?: AegTreeItem): Promise { + if (element) return element.children; + const [receiptCount, library] = await Promise.all([ + countExperienceReceipts(), + readBundledVerifiedLibrary(this.extensionUri) + ]); + const coverage = summarizeVerifiedLibraryCoverage(library.experiences); + const advanced = [ new AegTreeItem( - 'Try a verified experience', - 'task → match → recovery capsule', - 'library', - {command: 'aeg.tryVerifiedExperience', title: 'Try verified experience'} - ), - new AegTreeItem( - 'Open the transfer challenge', - 'bundled zero-cold-start demo', + 'Bundled transfer challenge', + 'synthetic workflow demonstration', 'lightbulb-autofix', {command: 'aeg.openVerifiedExperienceDemo', title: 'Open challenge'} ), @@ -98,28 +119,85 @@ class PlaywrightViewProvider implements vscode.TreeDataProvider { {command: 'aeg.diagnosePlaywrightFailure', title: 'Diagnose'} ), new AegTreeItem( - 'Mark latest outcome', + 'Mark latest Playwright outcome', 'resolved or unresolved', 'pass', {command: 'aeg.verifyLatestExperience', title: 'Verify'} ), new AegTreeItem( - 'Local experience receipts', - `${count} recorded`, + 'Local Playwright receipts', + `${receiptCount} recorded`, 'history', {command: 'aeg.showExperiences', title: 'Show experiences'} ), new AegTreeItem( - 'Run public repair lab', - 'baseline vs AEG-assisted', + 'Run public Repair Lab', + 'legacy controlled experiment runner', 'beaker', {command: 'aeg.runPublicRepairLab', title: 'Run repair lab'} ), new AegTreeItem( - 'Privacy', - 'local only · no upload', - 'shield', - {command: 'aeg.openGettingStarted', title: 'Getting started'} + 'Discover workspace skills', + 'legacy local skill discovery', + 'search', + {command: 'aeg.discoverSkills', title: 'Discover skills'} + ), + new AegTreeItem( + 'Recommend a workspace skill', + 'legacy local skill ranking', + 'wand', + {command: 'aeg.recommendSkill', title: 'Recommend skill'} + ), + new AegTreeItem( + 'Rate a workspace skill', + 'legacy local skill feedback', + 'star-empty', + {command: 'aeg.rateSkill', title: 'Rate skill'} + ), + new AegTreeItem( + 'Show skill metrics', + 'legacy local metrics', + 'graph', + {command: 'aeg.showSkillMetrics', title: 'Show metrics'} + ), + new AegTreeItem( + 'Legacy verified-experience alias', + 'backward-compatible command ID', + 'debug-step-back', + {command: 'aeg.tryVerifiedExperience', title: 'Compatibility alias'} + ), + new AegTreeItem( + 'Legacy getting started', + 'Playwright and Repair Lab reference', + 'file-text', + {command: 'aeg.openGettingStarted', title: 'Legacy getting started'} + ) + ]; + return [ + new AegTreeItem( + 'Start with Verified Experience', + 'search → inspect → hand off → validate → rate', + 'library', + {command: 'aeg.startWithVerifiedExperience', title: 'Start with Verified Experience'} + ), + new AegTreeItem( + 'Verified library coverage', + `${coverage.verifiedRecordCount} records · ${coverage.families.length} task families`, + 'book', + {command: 'aeg.showVerifiedCoverage', title: 'Show verified coverage'} + ), + new AegTreeItem( + 'Guided walkthrough', + 'complete the proof loop without the README', + 'map', + {command: 'aeg.openFounderWalkthrough', title: 'Open walkthrough'} + ), + new AegTreeItem( + 'Advanced', + 'Playwright · Repair Lab · skills · legacy', + 'tools', + undefined, + advanced ) ]; } @@ -127,12 +205,22 @@ class PlaywrightViewProvider implements vscode.TreeDataProvider { export function activate(context: vscode.ExtensionContext): void { const status = vscode.window.createStatusBarItem(vscode.StatusBarAlignment.Left, 100); - status.text = '$(library) AEG Experience'; - status.command = 'aeg.tryVerifiedExperience'; - status.tooltip = 'Retrieve verified debugging experience before starting from scratch'; + status.text = PRIMARY_ENTRY_STATUS_TEXT; + status.command = 'aeg.startWithVerifiedExperience'; + status.tooltip = 'Start with Verified Experience · 2 records · 2 task families · local only'; status.show(); - const viewProvider = new PlaywrightViewProvider(); + const firstRunStorage: FounderFirstRunStorage = { + get: () => context.globalState.get(FOUNDER_FIRST_RUN_MARKER_KEY), + update: marker => context.globalState.update(FOUNDER_FIRST_RUN_MARKER_KEY, marker) + }; + const openFounderWalkthrough = () => vscode.commands.executeCommand( + 'workbench.action.openWalkthrough', + FOUNDER_WALKTHROUGH_ID, + false + ); + + const viewProvider = new PlaywrightViewProvider(context.extensionUri); const tree = vscode.window.createTreeView('aeg.playwright', {treeDataProvider: viewProvider}); const notifiedArtifacts = new Set(); const watcher = vscode.workspace.createFileSystemWatcher('**/test-results/**/*'); @@ -158,6 +246,10 @@ export function activate(context: vscode.ExtensionContext): void { status, tree, watcher, + vscode.commands.registerCommand( + 'aeg.startWithVerifiedExperience', + () => tryVerifiedExperience(context.extensionUri) + ), vscode.commands.registerCommand( 'aeg.tryVerifiedExperience', () => tryVerifiedExperience(context.extensionUri) @@ -166,6 +258,21 @@ export function activate(context: vscode.ExtensionContext): void { 'aeg.openVerifiedExperienceDemo', () => tryVerifiedExperience(context.extensionUri, VERIFIED_EXPERIENCE_DEMO) ), + vscode.commands.registerCommand( + 'aeg.showVerifiedCoverage', + () => showVerifiedCoverage(context.extensionUri) + ), + vscode.commands.registerCommand( + 'aeg.openFounderWalkthrough', + async () => { + const result = await reopenFounderWalkthrough(firstRunStorage, openFounderWalkthrough); + if (result === 'failed') { + void vscode.window.showWarningMessage( + 'AEG could not open the walkthrough. Use the visible “AEG: Start here” status-bar action to begin.' + ); + } + } + ), vscode.commands.registerCommand( 'aeg.diagnosePlaywrightFailure', (uri?: vscode.Uri) => diagnosePlaywrightFailure(viewProvider, uri) @@ -182,6 +289,21 @@ export function activate(context: vscode.ExtensionContext): void { vscode.commands.registerCommand('aeg.rateSkill', rateSkill), vscode.commands.registerCommand('aeg.showSkillMetrics', showSkillMetrics) ); + + // Deferred startup activation plus fire-and-forget opening keeps VS Code startup non-blocking. + void openFounderWalkthroughOnFirstRun({ + hasWorkspace: Boolean(vscode.workspace.workspaceFolders?.length), + storage: firstRunStorage, + openSurface: openFounderWalkthrough + }).then(result => { + if (result === 'failed') { + console.warn('AEG first-run walkthrough did not open; the status-bar entry remains available.'); + } else if (result === 'opened-unpersisted') { + console.warn('AEG first-run walkthrough opened, but its global first-run marker was not persisted.'); + } + }).catch(error => { + console.warn('AEG first-run onboarding failed safely; the status-bar entry remains available.', error); + }); } export function deactivate(): void { @@ -194,37 +316,28 @@ async function tryVerifiedExperience(extensionUri: vscode.Uri, presetTask?: stri ? redactSensitiveText(editor.document.getText(editor.selection), 1_000) : ''; const task = presetTask ?? await vscode.window.showInputBox({ - title: 'AEG: Try a Verified Experience', - prompt: 'Describe the task, error, or issue. AEG ranks only bundled verified records; it does not upload this text.', + title: 'AEG: Start with Verified Experience', + prompt: 'Enter a task or use the selected error text. AEG searches two bundled verified records locally and may correctly abstain.', value: selectedText, placeHolder: 'A protocol wrapper still uses a stale socket after stream ownership moved' }); if (!task?.trim()) return; - const libraryUri = vscode.Uri.joinPath(extensionUri, 'verified-experiences', 'verified.json'); - let raw: string; - try { - raw = Buffer.from(await vscode.workspace.fs.readFile(libraryUri)).toString('utf8'); - } catch { - void vscode.window.showErrorMessage('The bundled verified-experience library is missing. Reinstall AEG v0.1.5.'); + let session = beginProofLoop(task, `proof-${Date.now()}`); + const library = await readBundledVerifiedLibrary(extensionUri); + if (!library.experiences.length && library.malformed.some(item => item.includes('missing'))) { + void vscode.window.showErrorMessage('The bundled verified-experience library is missing. Reinstall AEG v0.1.6.'); return; } - - const library = loadVerifiedExperienceLibrary(raw); if (library.malformed.length) { void vscode.window.showWarningMessage(`AEG excluded ${library.malformed.length} malformed verified-experience record(s).`); } const matches = rankVerifiedExperiences(task, library.experiences); if (!matches.length) { const nearMatch = rankVerifiedExperiences(task, library.experiences, Number.EPSILON, 1)[0]; - if (nearMatch) { - void vscode.window.showInformationMessage(describeBelowThresholdMatch(nearMatch)); - } else { - void vscode.window.showInformationMessage( - `AEG found no lexical match (score 0; threshold ${VERIFIED_EXPERIENCE_RETRIEVAL_THRESHOLD.toFixed(4)}). ` - + 'No candidate or fallback guidance was injected.' - ); - } + session = transitionProofLoop(session, {type: 'abstain'}); + await vscode.commands.executeCommand('setContext', 'aeg.proofLoopAbstained', true); + showVerifiedExperienceAbstention(session, library.experiences, nearMatch); return; } @@ -238,46 +351,108 @@ async function tryVerifiedExperience(extensionUri: vscode.Uri, presetTask?: stri match })), { - title: 'Verified experience matches', - placeHolder: 'Choose a verified record to inspect before your coding agent starts', + title: `Step 2 of 5 · ${library.experiences.length} verified records searched`, + placeHolder: 'Select a verified record to inspect; this does not claim it will solve the task', matchOnDescription: true, matchOnDetail: true } ); if (!picked) return; - await showVerifiedExperiencePanel(task, picked.match); + session = transitionProofLoop(session, {type: 'match', match: picked.match}); + session = transitionProofLoop(session, {type: 'inspect'}); + await vscode.commands.executeCommand('setContext', 'aeg.proofLoopInspected', true); + await showVerifiedExperiencePanel(session, picked.match); +} + +async function readBundledVerifiedLibrary(extensionUri: vscode.Uri) { + const libraryUri = vscode.Uri.joinPath(extensionUri, 'verified-experiences', 'verified.json'); + try { + const raw = Buffer.from(await vscode.workspace.fs.readFile(libraryUri)).toString('utf8'); + return loadVerifiedExperienceLibrary(raw); + } catch { + return {experiences: [], malformed: ['bundled verified-experience library is missing']}; + } } -async function showVerifiedExperiencePanel(task: string, match: VerifiedExperienceMatch): Promise { +async function showVerifiedExperiencePanel( + initialSession: ProofLoopSession, + match: VerifiedExperienceMatch +): Promise { const panel = vscode.window.createWebviewPanel( 'aegVerifiedExperience', - 'AEG: Verified Experience', + 'AEG: Verified Experience Proof Loop', vscode.ViewColumn.Beside, {enableScripts: true} ); + let session = initialSession; + let feedbackPending = false; const capsule = generateRecoveryCapsule(match); - panel.webview.html = verifiedExperienceHtml(match, capsule, panel.webview); + panel.webview.html = verifiedExperienceHtml(session, match, capsule, panel.webview); panel.webview.onDidReceiveMessage(async message => { if (message?.command === 'copy') { + if (session.stage !== 'inspected') return; + const copied = transitionProofLoop(session, {type: 'copy'}); await vscode.env.clipboard.writeText(capsule); - void vscode.window.showInformationMessage('Verified-experience recovery capsule copied. Validate it against the local repository before applying changes.'); + session = copied; + await vscode.commands.executeCommand('setContext', 'aeg.proofLoopCopied', true); + await postProofLoopState(panel, session); + void vscode.window.showInformationMessage( + 'Capsule copied. Open VS Code Chat, paste it into the chat input with your task, press Enter, then validate the result with focused and regression checks.' + ); + return; + } + if (message?.command === 'validate' && isValidationOutcome(message.outcome)) { + if (session.stage !== 'copied') return; + session = transitionProofLoop(session, {type: 'validate', outcome: message.outcome}); + await vscode.commands.executeCommand('setContext', 'aeg.proofLoopValidated', true); + await postProofLoopState(panel, session); return; } if (message?.command === 'rate' && isExperienceRating(message.rating)) { - const saved = await writeVerifiedExperienceFeedback(task, match, message.rating); - if (saved) void vscode.window.showInformationMessage(`AEG recorded “${message.rating}” feedback locally.`); + if (session.stage !== 'validated' || feedbackPending) return; + const rated = transitionProofLoop(session, {type: 'rate', rating: message.rating}); + feedbackPending = true; + try { + const saved = await writeVerifiedExperienceFeedback(rated, match); + if (saved) { + session = rated; + await vscode.commands.executeCommand('setContext', 'aeg.proofLoopRated', true); + await postProofLoopState(panel, session); + void vscode.window.showInformationMessage( + `AEG recorded “${message.rating}” feedback for ${match.experience.id} locally.` + ); + } + } finally { + feedbackPending = false; + } } }); } +async function postProofLoopState( + panel: vscode.WebviewPanel, + session: ProofLoopSession +): Promise { + await panel.webview.postMessage({ + type: 'proof-loop-state', + stage: session.stage, + step: proofLoopStep(session.stage), + validationOutcome: session.validationOutcome, + rating: session.rating + }); +} + function isExperienceRating(value: unknown): value is ExperienceRating { return value === 'helpful' || value === 'partially-helpful' || value === 'irrelevant' || value === 'harmful'; } +function isValidationOutcome(value: unknown): value is ValidationOutcome { + return value === 'passed' || value === 'partially-passed' || value === 'failed' || value === 'not-applied'; +} + async function writeVerifiedExperienceFeedback( - task: string, - match: VerifiedExperienceMatch, - rating: ExperienceRating + session: ProofLoopSession, + match: VerifiedExperienceMatch ): Promise { const root = workspaceRoot(); if (!root) { @@ -297,12 +472,16 @@ async function writeVerifiedExperienceFeedback( } catch { // A missing file is the expected first-use state. } + if (!session.rating || !session.validationOutcome) return false; const feedback: ExperienceFeedback = { - schemaVersion: '1.0.0', + schemaVersion: '1.1.0', recordedAt: new Date().toISOString(), + proofLoopSessionId: session.id, experienceId: match.experience.id, - taskSummary: redactSensitiveText(task, 500), - rating, + experienceTask: match.experience.task, + taskSummary: redactSensitiveText(session.query, 500), + rating: session.rating, + validationOutcome: session.validationOutcome, retrievalScore: match.score, localOnly: true }; @@ -314,6 +493,7 @@ async function writeVerifiedExperienceFeedback( } function verifiedExperienceHtml( + session: ProofLoopSession, match: VerifiedExperienceMatch, capsule: string, webview: vscode.Webview @@ -332,27 +512,90 @@ body{padding:28px;max-width:920px;margin:auto;font:14px/1.55 var(--vscode-font-f .badge{display:inline-block;padding:4px 9px;border-radius:999px;background:var(--vscode-badge-background);color:var(--vscode-badge-foreground)} .card,.guardrail{margin:18px 0;padding:18px;border:1px solid var(--vscode-widget-border);border-radius:10px;background:var(--vscode-editor-background)} .guardrail{border-left:4px solid var(--vscode-editorWarning-foreground)} +.query{border-left:4px solid var(--vscode-focusBorder)} +.pending{opacity:.6}.complete{border-left:4px solid var(--vscode-testing-iconPassed)} .grid{display:grid;grid-template-columns:1fr 1fr;gap:16px} h1{line-height:1.15} h2{margin-top:26px} li{margin:6px 0} table{width:100%;border-collapse:collapse}th,td{text-align:left;padding:7px;border-bottom:1px solid var(--vscode-widget-border)} pre{white-space:pre-wrap;padding:14px;background:var(--vscode-textCodeBlock-background);overflow:auto} button{margin:6px 8px 0 0;padding:8px 12px;color:var(--vscode-button-foreground);background:var(--vscode-button-background);border:0;border-radius:4px;cursor:pointer} button.secondary{color:var(--vscode-button-secondaryForeground);background:var(--vscode-button-secondaryBackground)} +button:disabled{opacity:.45;cursor:not-allowed}.status{font-weight:600} @media(max-width:650px){.grid{grid-template-columns:1fr}} +Step 2 of 5 · verified match +

Inspect evidence before handoff

+

Original query

${escapeHtml(session.query)}

Session ${escapeHtml(session.id)} · stays local

${Math.round(match.score * 100)}% weighted match · verification ${escapeHtml(experience.verification.status)} -

${escapeHtml(experience.task)}

+

${escapeHtml(experience.task)}

${escapeHtml(experience.id)}

Guidance, not a guaranteed answer. Inspect the local code, reproduce the failure, and validate any repair with focused and regression tests.

Reusable lessons

    ${list(experience.lessons)}

Recommended use cases

    ${list(experience.reuse.recommendedFor)}

Constraints

    ${list(experience.constraints)}

Limitations

    ${list(experience.limitations)}

Why this matched

${evidence}
FieldTask phraseExperience phraseLexicalWeighted

Provenance and outcome

Outcome: ${escapeHtml(experience.outcome)}. Public source: ${escapeHtml(source.repository)} · ${escapeHtml(source.license)} · ${escapeHtml(source.benchmark)}. Experiment artifact: ${escapeHtml(experience.provenance.experimentEvidence.artifact)}.

-

Compact recovery capsule

${escapeHtml(capsule)}

-

Was this useful?

-

Ratings are written only to this workspace under .aeg/. AEG v0.1.5 does not upload task text, code, capsules, or ratings.

- +

Step 3 · Guarded capsule handoff

${escapeHtml(capsule)}

+

Step 4 · Validate the outcome

For the original query above, run a focused check and relevant regression checks. What objective result did you observe?

+

Step 5 · Rate this retrieval

Rate ${escapeHtml(experience.id)} for the original query after recording the validation outcome.

+

Feedback is written only to this workspace under .aeg/verified-experience-feedback.json. AEG v0.1.6 does not upload task text, code, prompts, logs, capsules, ratings, receipts, or private data.

+ `; } +function showVerifiedExperienceAbstention( + session: ProofLoopSession, + experiences: ReturnType['experiences'], + nearMatch?: VerifiedExperienceMatch +): void { + const panel = vscode.window.createWebviewPanel( + 'aegVerifiedExperienceAbstention', + 'AEG: No Relevant Verified Experience', + vscode.ViewColumn.Beside, + {enableScripts: true} + ); + const coverage = summarizeVerifiedLibraryCoverage(experiences); + const explanation = nearMatch + ? describeBelowThresholdMatch(nearMatch) + : `AEG found no lexical match (score 0; threshold ${VERIFIED_EXPERIENCE_RETRIEVAL_THRESHOLD.toFixed(4)}). No candidate or fallback guidance was injected.`; + const nonce = `${Date.now()}`; + panel.webview.html = `

Step 2 of 5 · retrieval outcome

No relevant verified experience

This is a correct product outcome, not an error. AEG did not force a weak match or inject generic fallback guidance.

Original query

${escapeHtml(session.query)}

Why AEG abstained

${escapeHtml(explanation)}

Current coverage

${coverage.verifiedRecordCount} verified records across ${coverage.families.length} task families:

    ${coverage.families.map(family => `
  • ${escapeHtml(family.label)} — ${escapeHtml(family.description)}
    • `).join('')}

Tasks outside those narrow families will often and correctly abstain.

No code, task text, prompt, log, rating, receipt, or private data was uploaded.

`; + panel.webview.onDidReceiveMessage(async message => { + if (message?.command === 'retry') { + panel.dispose(); + await vscode.commands.executeCommand('aeg.startWithVerifiedExperience'); + } else if (message?.command === 'coverage') { + await vscode.commands.executeCommand('aeg.showVerifiedCoverage'); + } + }); +} + +async function showVerifiedCoverage(extensionUri: vscode.Uri): Promise { + const library = await readBundledVerifiedLibrary(extensionUri); + const coverage = summarizeVerifiedLibraryCoverage(library.experiences); + const families = coverage.families.map(family => + `- **${family.label}** (${family.recordCount} record): ${family.description}` + ).join('\n'); + const document = await vscode.workspace.openTextDocument({ + language: 'markdown', + content: `# AEG verified-library coverage\n\n` + + `**${coverage.verifiedRecordCount} verified records across ${coverage.families.length} narrow task families.**\n\n${families}\n\n` + + `AEG uses deterministic lexical ranking with a fixed ${VERIFIED_EXPERIENCE_RETRIEVAL_THRESHOLD.toFixed(4)} threshold. ` + + `Below-threshold and zero-score results are shown as **No relevant verified experience**. ` + + `That abstention is expected and injects no candidate or generic fallback guidance.\n\n` + + `Verified means the source outcome was objectively checked. It does not mean retrieval improves success, speed, cost, or generalization.\n` + }); + await vscode.window.showTextDocument(document, {preview: true}); +} + async function diagnosePlaywrightFailure( viewProvider: PlaywrightViewProvider, preferredArtifact?: vscode.Uri @@ -656,7 +899,7 @@ function recoveryHtml(

Verify the outcome

After trying the playbook and re-running the test, record the objective result.

-

Local only: AEG v0.1.5 does not upload code, logs, or experience receipts.

+

Local only: AEG v0.1.6 does not upload code, logs, or experience receipts.