From f8835c0c4533d35dcb16193b20587453967b9464 Mon Sep 17 00:00:00 2001 From: Yao Li Date: Tue, 11 Aug 2026 22:42:54 -0700 Subject: [PATCH 01/13] Build v0.1.6 founder-ready proof loop --- .github/workflows/repair-lab.yml | 2 + README.md | 53 +-- docs/60-second-demo.md | 57 ++- experiments/v0.1.6-product-proof/README.md | 57 +++ experiments/v0.1.6-product-proof/STATUS.md | 48 +++ .../v0.1.6-product-proof/UX-ACCEPTANCE.md | 34 ++ experiments/v0.1.6-product-proof/UX-AUDIT.md | 16 + .../v0.1.6-product-proof/generic-advice.txt | 1 + .../v0.1.6-product-proof/protocol.json | 142 +++++++ .../v0.1.6-product-proof/protocol.schema.json | 216 +++++++++++ .../v0.1.6-product-proof/result.schema.json | 188 +++++++++ integrations/vscode/CHANGELOG.md | 14 + integrations/vscode/README.md | 208 ++++------ integrations/vscode/package-lock.json | 5 +- integrations/vscode/package.json | 166 +++++--- integrations/vscode/src/extension.ts | 360 ++++++++++++++---- integrations/vscode/src/proofLoop.ts | 84 ++++ integrations/vscode/src/verifiedExperience.ts | 55 ++- integrations/vscode/test/core.test.js | 87 ++++- .../vscode/test/experiment-schema.test.js | 127 ++++++ integrations/vscode/walkthrough/01-start.md | 9 + integrations/vscode/walkthrough/02-inspect.md | 5 + integrations/vscode/walkthrough/03-handoff.md | 7 + .../vscode/walkthrough/04-validate.md | 5 + integrations/vscode/walkthrough/05-rate.md | 10 + 25 files changed, 1638 insertions(+), 318 deletions(-) create mode 100644 experiments/v0.1.6-product-proof/README.md create mode 100644 experiments/v0.1.6-product-proof/STATUS.md create mode 100644 experiments/v0.1.6-product-proof/UX-ACCEPTANCE.md create mode 100644 experiments/v0.1.6-product-proof/UX-AUDIT.md create mode 100644 experiments/v0.1.6-product-proof/generic-advice.txt create mode 100644 experiments/v0.1.6-product-proof/protocol.json create mode 100644 experiments/v0.1.6-product-proof/protocol.schema.json create mode 100644 experiments/v0.1.6-product-proof/result.schema.json create mode 100644 integrations/vscode/src/proofLoop.ts create mode 100644 integrations/vscode/test/experiment-schema.test.js create mode 100644 integrations/vscode/walkthrough/01-start.md create mode 100644 integrations/vscode/walkthrough/02-inspect.md create mode 100644 integrations/vscode/walkthrough/03-handoff.md create mode 100644 integrations/vscode/walkthrough/04-validate.md create mode 100644 integrations/vscode/walkthrough/05-rate.md diff --git a/.github/workflows/repair-lab.yml b/.github/workflows/repair-lab.yml index 95420a0..02d188f 100644 --- a/.github/workflows/repair-lab.yml +++ b/.github/workflows/repair-lab.yml @@ -6,6 +6,7 @@ on: - "experiences/**" - "experiments/public-repair-lab/**" - "experiments/natural-transfer-benchmark/**" + - "experiments/v0.1.6-product-proof/**" - "integrations/vscode/**" - "scripts/**" - "references/trace_schema.md" @@ -22,6 +23,7 @@ on: - "experiences/**" - "experiments/public-repair-lab/**" - "experiments/natural-transfer-benchmark/**" + - "experiments/v0.1.6-product-proof/**" - "integrations/vscode/**" - "scripts/**" - "references/trace_schema.md" diff --git a/README.md b/README.md index 3df3ea7..3818ce9 100644 --- a/README.md +++ b/README.md @@ -3,11 +3,12 @@ Agent Experience Graph helps coding agents retrieve verified debugging experience instead of solving every problem from scratch. -The v0.1.5 developer preview turns a task or error into an explainable match -from a small verified-only public library, then produces a compact recovery -capsule for the coding agent. The capsule preserves lessons, failed approaches, -constraints, limitations, and public provenance. It is guidance to validate, -not a guaranteed answer. +The v0.1.6 product-proof release gives the VS Code extension one honest path: +a task or error becomes an explainable verified match or explicit abstention; +an above-threshold match exposes evidence and limitations before a guarded +capsule is copied; the user then records an objective validation outcome and +local usefulness rating. It retrieves guidance and does not automatically +solve, send, or run the task. Think of it like a shared memory of successful work patterns. When an agent starts a new task, it can look at previous tasks, see which parts were similar, and learn which tools, skills, and approaches helped before. @@ -28,18 +29,22 @@ Open the investor- and partner-friendly living pitch: The pitch covers the vision, problem, architecture, initial product, early progress, business model, roadmap, founder, and current ask. -## Try a Verified Experience in VS Code +## v0.1.6 VS Code quick start Install [AEG from the Visual Studio Marketplace](https://marketplace.visualstudio.com/items?itemName=AgentExperienceGraph.agent-experience-graph), -then use this three-step quick start: +then use the first-run walkthrough or this quick start: -1. Run **AEG: Try a Verified Experience** and describe a task or select an error. -2. Inspect why a verified record matched, then copy its guarded capsule before - the coding agent begins. -3. Validate the repair and rate the retrieval locally. +1. Run **AEG: Start with Verified Experience** and describe a task or select an error. +2. Inspect the score, matching phrases, provenance, constraints, and limitations—or accept **No relevant verified experience** as a correct outcome. +3. Copy an above-threshold guarded capsule. Open VS Code Chat, paste it into the chat input with the original task, and press Enter. +4. Run focused and regression checks, record the observed outcome, then rate the selected experience locally. -For zero-cold-start onboarding, run **AEG: Open Verified Experience -Challenge**. It uses a bundled synthetic transfer fixture and openly reports +The sidebar shows the honest boundary up front: **2 verified records · 2 task +families**. Playwright diagnosis, Repair Lab, skill discovery, the synthetic +challenge, and legacy commands remain available under **Advanced**. + +For zero-cold-start onboarding, use the walkthrough's bundled guided task. The +Advanced **Open Bundled Transfer Challenge** command uses a synthetic fixture and reports that its prior controlled pair found no repair-path or outcome improvement and higher assisted token and wall-time cost. See [`experiments/verified-experience-challenge/`](experiments/verified-experience-challenge/). @@ -67,9 +72,10 @@ higher assisted token and wall-time cost. See ## 60-Second Demo -Run **AEG: Open Verified Experience Challenge** from the VS Code command -palette. The demo shows task entry, verified-only retrieval, weighted match -evidence, a compact guarded capsule, and local usefulness feedback. +Run **AEG: Start with Verified Experience** from the VS Code command palette +and use the bundled task in the walkthrough. The demo shows task entry, +verified-only retrieval, weighted match evidence, a compact guarded capsule, +explicit paste instructions, validation, and local usefulness feedback. Use [`docs/60-second-demo.md`](docs/60-second-demo.md) for a short meeting talk track. @@ -85,7 +91,8 @@ Agent Experience Graph gives agents a way to ask: - Which tools failed or wasted time? - What should I watch out for? -This can make agents more reliable, faster to start, and easier to improve over time. +Whether this makes agents more reliable or efficient is an open question that +requires controlled evidence beyond the current two-record library. ## A Non-Technical Example @@ -112,8 +119,8 @@ It is intentionally runtime-neutral: - `scripts/recommend_traces.py` ranks similar traces and recommends reusable skills/tools. - `references/trace_schema.md` defines the trace data contract. - `experiences/verified.json` stores sanitized, executed, and objectively verified shared experiences. -- `integrations/vscode/` exposes the verified-experience challenge and the - existing Playwright workflow. +- `integrations/vscode/` exposes the v0.1.6 verified-experience golden path and + keeps prior Playwright, Repair Lab, and skill tools under Advanced. - `experiments/verified-experience-challenge/` supplies a transparent bundled transfer demo. - `experiments/public-repair-lab/` runs the first baseline-versus-AEG public bug repair experiment. @@ -183,10 +190,10 @@ This project may be useful for: Agent Experience Graph is designed around sanitized traces, not raw logs. -The v0.1.5 extension bundles its public verified library and performs retrieval -locally. It does not upload task text, code, logs, recovery capsules, receipts, -or usefulness ratings. Local ratings are stored under `.aeg/`; review or ignore -that directory before committing it. +The v0.1.6 extension bundles its public verified library and performs retrieval +locally. It does not upload task text, code, prompts, logs, recovery capsules, +receipts, ratings, or private data. Local validation outcomes and ratings are +stored under `.aeg/`; review or ignore that directory before committing it. That means shared traces should not contain: diff --git a/docs/60-second-demo.md b/docs/60-second-demo.md index 671492f..bfe29c8 100644 --- a/docs/60-second-demo.md +++ b/docs/60-second-demo.md @@ -1,54 +1,43 @@ -# 60-Second Demo Talk Track +# 60-second v0.1.6 demo ## Goal -Show one memorable moment: +Show one honest loop: -> A developer gives AEG a debugging task and retrieves a verified, explainable -> recovery capsule before the coding agent starts from scratch. +> Task or error → verified match or abstention → inspect evidence → guarded handoff → validate → local feedback. -This demo is intentionally lightweight. It is not meant to prove the final architecture. It is meant to make the product abstraction easy to understand in a short meeting. +This is an interaction demo. It does not prove that retrieval improves correctness, success, speed, cost, adoption, or generalization. -## Demo Script +## Talk track -Install AEG v0.1.5, open the repository in VS Code, and run **AEG: Open -Verified Experience Challenge**. +Install AEG v0.1.6, open a test workspace in VS Code, and run **AEG: Start with Verified Experience**. -Say: +Point out the sidebar first: -> This public wrapper still uses a stale resource after ownership moved behind a -> protocol layer. Before the coding agent starts, AEG searches only its bundled -> verified records. +> There is one primary action. Coverage is visible: two verified records in two narrow task families. Playwright, Repair Lab, skill discovery, the synthetic challenge, and legacy commands remain under Advanced. -Select the TR-04 result and point to **Why this matched**. +Use this task: -Then say: +```text +Keepalive control fails after active stream ownership moved behind a protocol object; repair the public wrapper so it delegates through the protocol without using its stale socket field. +``` -> AEG exposes the exact fields and weighted lexical evidence behind the match. -> The experience records the original failed client-side approach, the recovery -> principle, constraints, limitations, and public provenance. +Select the TR-04 result and say: -Click **Copy capsule**. +> AEG exposes the exact matching phrases and weighted lexical score, then shows the objectively checked source outcome, public provenance, constraints, and limitations. Verified describes the source record; it does not guarantee this task. -> The capsule is guidance, not an answer guarantee. It tells the agent to inspect -> the local code, reproduce the failure, and validate the patch. +Select **Copy capsule** and point to the instructions: -Point to the rating buttons: +> AEG copies guarded guidance and tells me exactly where to paste it. It does not call a private chat API, submit a prompt, or run an agent. -> After validation, the developer records whether the experience was helpful, -> partially helpful, irrelevant, or harmful. That feedback stays in `.aeg/`. +Point to the disabled-then-enabled validation and rating steps: -## Core Message +> I must record an observed focused/regression-check outcome before I can rate this experience. The query, selected experience, validation result, and rating stay together in `.aeg/` and are not uploaded. -AEG helps coding agents retrieve verified debugging experience instead of -solving every problem from scratch. The current evidence is deliberately -narrow: the prior pair for this synthetic challenge found the same successful -repair in both arms and higher assisted token and wall-time cost. +If time permits, start again with `Change the website navigation background from white to blue and increase the logo size.` Show **No relevant verified experience**: -## Questions to Ask +> Abstention is a correct outcome. AEG shows the score and threshold, discloses current coverage, and injects no generic fallback. -- What is the right abstraction for reusable agent skills? -- Are failure patterns more reusable than successful workflows? -- Should skills be prompts, tools, workflows, policies, or all of the above? -- Where would this fit in the current agent developer ecosystem? -- What would make this useful for real developers rather than just an interesting demo? +## Evidence boundary + +The bundled task is a synthetic transfer demonstration. Its prior controlled pair produced the same successful patch and repair path in both arms while assisted token use and wall time were higher. It demonstrates discoverability and interaction, not performance benefit. diff --git a/experiments/v0.1.6-product-proof/README.md b/experiments/v0.1.6-product-proof/README.md new file mode 100644 index 0000000..f2f7a3f --- /dev/null +++ b/experiments/v0.1.6-product-proof/README.md @@ -0,0 +1,57 @@ +# AEG v0.1.6 product-proof preparation + +Status: **prepared only; not frozen; 0/3 arms executed**. + +This directory prepares one bounded three-arm test of the v0.1.6 product hypothesis: whether a top-1 verified capsule changes the outcome or effort on a naturally occurring public repair when compared with both no added context and fixed generic debugging advice. + +It does not authorize or run a model, spend money, select a target, inspect a human patch, or change the verified library. + +## Three arms + +1. **Baseline:** the frozen public task, code, failure, budget, and objective checks; no AEG records or added advice. +2. **Fixed generic:** identical inputs plus [`generic-advice.txt`](generic-advice.txt); no AEG record access. +3. **AEG top-1:** identical inputs plus only the automatically retrieved and frozen top-1 guarded capsule. + +Model, settings, prompt template, budget, environment, focused checks, regression checks, and evaluation are identical. The intentional context difference is the arm definition above. + +## Freeze gate before any execution + +A separate authorized phase must freeze all currently `null` fields in [`protocol.json`](protocol.json) before an arm can run: + +- a naturally occurring target from a predeclared candidate pool in a public repository with an explicit license; +- repository and buggy commit, task hash, objective check hashes, and freeze time before inspecting its human patch; +- a source experience whose timestamp predates the target; +- verified-library hash, fixed 0.0500 threshold, retrieval score, top-1 experience, and guarded-capsule hash; +- identical model, settings, prompt template, budget, oracle, and randomized arm order. + +Exclude eBay, private, internal, proprietary, credentialed, or capsule-derived targets. If no verified result clears the frozen threshold, record a retrieval abstention and stop without executing any arm. + +Each arm must use a separate sanitized repository and worktree directory with no shared Git metadata, writable cache, AEG access for controls, human patch, other-arm artifacts, or evaluator feedback. + +## Results + +[`result.schema.json`](result.schema.json) requires: + +- success, attempts, completed sanitized commands, and test executions; +- focused and regression check results; +- non-cached tokens and duration; +- repository-relative files inspected and changed; +- patch hash and identical-setting hashes; +- limitations, protocol deviations, and a privacy attestation. + +Publish negative, neutral, abstention, and protocol-deviation outcomes. Do not publish code, prompts, task text, raw logs, ratings, receipts, or private data. Do not change thresholds or outcome rules after observing a result. + +## Validation only + +The extension test suite validates both experiment schemas and checks valid completed, abstention, missing-measurement, and wrong-arm-order cases: + +```bash +cd integrations/vscode +npm test +``` + +This command does not execute an experiment arm. + +## Evidence boundary + +AEG currently contains two verified records in two narrow task families. The bundled challenge demonstrates the interaction flow, not a performance benefit. Prior controlled and transfer evidence was neutral or negative. Nothing in this preparation supports a claim of improved success, speed, cost, PMF, adoption, or generalization. diff --git a/experiments/v0.1.6-product-proof/STATUS.md b/experiments/v0.1.6-product-proof/STATUS.md new file mode 100644 index 0000000..4dd749d --- /dev/null +++ b/experiments/v0.1.6-product-proof/STATUS.md @@ -0,0 +1,48 @@ +# AEG v0.1.6 product-proof status + +Last updated: 2026-08-11 (America/Los_Angeles). + +## Current phase + +Implementation and local validation complete; ready for the founder usability gate. Publication handoff is pending. Experiment status remains **prepared-not-frozen; 0/3 arms executed**. + +## Completed work + +- Audited the v0.1.5 workflow, required evidence, both dogfood decisions, the verified challenge, natural-transfer benchmark, and GitHub Issue #12. +- Preserved the two-record verified library and fixed 0.0500 retrieval threshold. +- Added one dominant verified-experience command, honest coverage, explicit abstention, guarded handoff instructions, enforced validation-before-rating, and local query/experience/outcome-linked feedback. +- Moved Playwright, Repair Lab, skill tools, the synthetic challenge, and legacy commands under **Advanced** without deleting command IDs. +- Added a five-step first-install walkthrough using supported VS Code contribution points. +- Prepared the non-executing baseline/fixed-generic/AEG-top-1 protocol, generic advice, strict schemas, UX audit, and founder acceptance gate. + +## Tests + +- Untouched v0.1.5 baseline: 20/20 extension tests passed; TypeScript compiled; baseline VSIX packaged. +- v0.1.6 extension, UX-transition, and schema suite: 29/29 tests passed; TypeScript compiled. +- Retrieval and verified-experience validation: 11/11 Python tests passed; two records validated with two unique IDs. +- Site regression suite: 8/8 tests passed. +- Patch whitespace and private-path/credential scans: passed. +- `experiences/verified.json` diff against `origin/main`: empty. +- VSIX content inspection: version 0.1.6, five walkthrough files, and exactly two verified records; packaged record hash matches the source library. + +## VSIX + +- Path: `integrations/vscode/agent-experience-graph-0.1.6.vsix` (local ignored build artifact) +- SHA-256: `8a5b53dfe5597ba42403edec5c780dbf352c5be3cf151cbff5558859e4b0f498` + +## Blockers + +- GitHub CLI authentication is invalid. Public read access worked; push and Draft PR creation may require `gh auth login` unless Git credentials independently permit the push. +- No product or experiment-design blocker is open. + +## Decisions + +- Use clipboard plus explicit Chat paste-and-run instructions. The documented editor-chat command is not a reliable normal-Chat handoff, and no private workbench command is used. +- Treat abstention as a successful retrieval decision and inject no generic fallback. +- Require a recorded validation outcome before usefulness feedback. +- Keep all feedback and receipts local; upload nothing from the extension. +- Do not select a target or execute an arm in v0.1.6. + +## Exact founder action required next + +After the final VSIX path is recorded here, clean-install that VSIX and perform the exact three-minute test in [`UX-ACCEPTANCE.md`](UX-ACCEPTANCE.md). Report each pass/fail item on the Draft PR. Do not run an experiment arm; target selection and protocol freeze require a separate explicit authorization after the usability gate passes. diff --git a/experiments/v0.1.6-product-proof/UX-ACCEPTANCE.md b/experiments/v0.1.6-product-proof/UX-ACCEPTANCE.md new file mode 100644 index 0000000..9706927 --- /dev/null +++ b/experiments/v0.1.6-product-proof/UX-ACCEPTANCE.md @@ -0,0 +1,34 @@ +# v0.1.6 founder UX acceptance + +This is a usability gate, not an experiment arm and not evidence of repair benefit. + +## Clean-install precondition + +1. Uninstall any existing **Agent Experience Graph** extension from VS Code. +2. Close every VS Code window. +3. Install the v0.1.6 VSIX with `code --install-extension /absolute/path/to/agent-experience-graph-0.1.6.vsix --force`. +4. Open a disposable public or local test workspace in a fresh VS Code window. + +## Exact three-minute founder test + +Start a three-minute timer after the AEG walkthrough appears. + +1. Confirm the walkthrough and sidebar make **AEG: Start with Verified Experience** the obvious first action, with **2 records · 2 task families** visible and legacy tools collapsed under **Advanced**. +2. Run the primary command with: `Keepalive control fails after active stream ownership moved behind a protocol object; repair the public wrapper so it delegates through the protocol without using its stale socket field.` +3. Select the above-threshold result. Confirm the panel keeps the original query visible and shows matching phrases, score, verified outcome, public provenance, constraints, limitations, and the “guidance, not a guaranteed answer” guardrail. +4. Select **Copy capsule**. Confirm the clipboard contains the capsule and the panel says exactly how to open VS Code Chat, where to paste it, and that AEG did not send or run it. +5. For this usability test, select **Did not apply**, then **Irrelevant**. Confirm `.aeg/verified-experience-feedback.json` links the original query summary, selected experience ID/task, validation outcome, rating, retrieval score, and local-only flag. +6. Start again with: `Change the website navigation background from white to blue and increase the logo size.` Confirm **No relevant verified experience** is presented as a correct outcome with score/threshold reasoning, coverage, and no injected fallback. + +## Pass criteria + +- The primary command is discoverable without opening a README. +- Every matched step is ordered and understandable without guessing the next action. +- The user cannot validate before copying or rate before validation. +- Match evidence and evidence limitations remain visible before handoff. +- Handoff uses only clipboard plus explicit instructions; no undocumented chat command is invoked. +- Feedback connects query, selected experience, observed validation outcome, and rating in a local workspace file. +- Abstention is calm, explicit, and informative. +- No network request, upload, model run, Repair Lab run, experiment arm, release, or Marketplace publish occurs. + +Any failed item keeps the release out of the founder usability gate. Do not reinterpret a usability pass as performance, adoption, PMF, or generalization evidence. diff --git a/experiments/v0.1.6-product-proof/UX-AUDIT.md b/experiments/v0.1.6-product-proof/UX-AUDIT.md new file mode 100644 index 0000000..08faf65 --- /dev/null +++ b/experiments/v0.1.6-product-proof/UX-AUDIT.md @@ -0,0 +1,16 @@ +# v0.1.5 first-user UX audit + +Audit date: 2026-08-11. Baseline: `origin/main` at `544874c`, extension version 0.1.5. + +The untouched baseline compiled, passed all 20 extension tests, and produced a local VSIX. The individual features worked, but a first-time user had no clearly dominant workflow: + +- **Try a Verified Experience**, **Open Verified Experience Challenge**, Playwright diagnosis, outcome marking, receipt history, Repair Lab, and skill tools appeared at similar prominence across the sidebar, command palette, editor context menu, view title, and status bar. +- The Activity Bar view was named **Verified Experience & Playwright**, so product identity and a legacy diagnosis vertical competed before the user entered a task. +- The primary verified card explained a match and copied a guarded capsule, but did not tell the user exactly which chat input to use or advance through an explicit validation state. +- Four usefulness buttons were available before any observed outcome was recorded. Feedback linked the query and experience, but not a validation result. +- The UI did not foreground the two-record/two-family coverage boundary. Abstention appeared as a transient notification, which could read like failure rather than calibrated retrieval. +- The README carried the only end-to-end explanation; there was no first-install walkthrough that tracked completion. + +The v0.1.6 decision is one reversible path: **Start with Verified Experience → inspect match or abstention → copy with explicit paste instructions → record objective validation → save local rating**. Playwright, Repair Lab, skill discovery, and all legacy command IDs remain available under **Advanced**. + +The supported VS Code command list documents `vscode.editorChat.start`, which starts editor chat, but does not document a stable command for an extension to open and prefill the normal Chat view. v0.1.6 therefore uses the supported clipboard API and tells the user exactly how to open Chat, paste, and run; it does not depend on a private workbench command. diff --git a/experiments/v0.1.6-product-proof/generic-advice.txt b/experiments/v0.1.6-product-proof/generic-advice.txt new file mode 100644 index 0000000..b9965f4 --- /dev/null +++ b/experiments/v0.1.6-product-proof/generic-advice.txt @@ -0,0 +1 @@ +Reproduce the failure before editing. Read the failing test, the nearby production code, and relevant history available in the frozen target. Form a small set of hypotheses and test the highest-signal one first. Make the smallest maintainable change that addresses the observed cause. Run the focused test after each repair attempt, then run the frozen regression checks. Do not modify tests, weaken assertions, inspect a human fix, or claim success without the objective checks. diff --git a/experiments/v0.1.6-product-proof/protocol.json b/experiments/v0.1.6-product-proof/protocol.json new file mode 100644 index 0000000..a2a7e33 --- /dev/null +++ b/experiments/v0.1.6-product-proof/protocol.json @@ -0,0 +1,142 @@ +{ + "$schema": "./protocol.schema.json", + "schemaVersion": "1.0.0", + "protocolId": "aeg-v0.1.6-product-proof", + "status": "prepared-not-frozen", + "executionAuthorized": false, + "evidenceBoundary": { + "verifiedRecordCount": 2, + "verifiedLibraryMutable": false, + "challengeEvidence": "interaction-only", + "priorTransferEvidence": "neutral-or-negative", + "claimsExcluded": [ + "improved-success", + "improved-speed", + "improved-cost", + "product-market-fit", + "adoption", + "generalization" + ] + }, + "arms": [ + { + "id": "baseline", + "label": "Baseline", + "additionalContext": null, + "aegLibraryAccess": false, + "targetInputs": "identical-frozen", + "objectiveOracle": "identical-frozen" + }, + { + "id": "fixed-generic", + "label": "Fixed generic debugging advice", + "additionalContext": "generic-advice.txt", + "aegLibraryAccess": false, + "targetInputs": "identical-frozen", + "objectiveOracle": "identical-frozen" + }, + { + "id": "aeg-top-1", + "label": "AEG top-1 verified capsule", + "additionalContext": "frozen-top-1-guarded-capsule", + "aegLibraryAccess": "top-1-only", + "targetInputs": "identical-frozen", + "objectiveOracle": "identical-frozen" + } + ], + "eligibility": { + "publicRepository": true, + "explicitLicense": true, + "excludedSources": [ + "ebay", + "private", + "internal", + "proprietary" + ], + "sourcePredatesTarget": true, + "targetFrozenBeforeHumanPatch": true, + "syntheticTargetForbidden": true, + "candidatePoolFrozenBeforeRetrieval": true + }, + "freeze": { + "requiredBeforeExecution": true, + "target": { + "repository": null, + "license": null, + "buggyCommitSha": null, + "taskHash": null, + "focusedTestHash": null, + "regressionTestHash": null, + "frozenAt": null + }, + "sourceExperience": { + "id": null, + "recordedAt": null, + "librarySha256": null, + "retrievalScore": null, + "retrievalThreshold": 0.05, + "capsuleSha256": null + }, + "controls": { + "model": null, + "settingsSha256": null, + "promptTemplateSha256": null, + "budgetSha256": null, + "oracleSha256": null, + "genericAdviceSha256": "8d90bc397194fa3a6c146914e36860ffe887a77fb8de63ed9e5212099fdbd022" + } + }, + "execution": { + "isolation": "separate sanitized repositories and worktree directories with no shared Git metadata or writable caches", + "freshContextPerArm": true, + "sharedCaches": false, + "armOrder": null, + "maxAttemptsPerArm": 3, + "maxDurationSecondsPerArm": 1500, + "humanPatchAvailableDuringArms": false, + "executeInThisRelease": false + }, + "measurements": [ + "success", + "attempts", + "commands", + "tests", + "nonCachedTokens", + "durationMs", + "filesInspected", + "filesChanged", + "patchHash" + ], + "evaluation": { + "focusedTestsRequired": true, + "regressionTestsRequired": true, + "sameOracleAcrossArms": true, + "thresholdChangesAfterResults": false, + "blindArmEvaluation": true, + "successDefinition": "The unchanged focused and regression checks pass without test edits or a protocol violation." + }, + "publication": { + "publishNegativeOutcomes": true, + "publishAbstentions": true, + "publishProtocolDeviations": true, + "forbiddenArtifacts": [ + "code", + "prompts", + "task-text", + "raw-logs", + "ratings", + "receipts", + "private-data" + ] + }, + "stopConditions": [ + "No verified result clears the frozen threshold.", + "The target is synthetic or constructed from a capsule.", + "The source experience does not predate the frozen target.", + "A public explicit license cannot be confirmed.", + "The human patch or another arm becomes visible.", + "Identical settings, budget, or objective oracles cannot be maintained.", + "Isolation, credentials, privacy, or authorization boundaries fail.", + "An experiment arm is requested before a separate execution authorization." + ] +} diff --git a/experiments/v0.1.6-product-proof/protocol.schema.json b/experiments/v0.1.6-product-proof/protocol.schema.json new file mode 100644 index 0000000..9dbf63d --- /dev/null +++ b/experiments/v0.1.6-product-proof/protocol.schema.json @@ -0,0 +1,216 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://aeg.local/experiments/v0.1.6-product-proof/protocol.schema.json", + "title": "AEG v0.1.6 product-proof protocol", + "type": "object", + "additionalProperties": false, + "required": [ + "schemaVersion", + "protocolId", + "status", + "executionAuthorized", + "evidenceBoundary", + "arms", + "eligibility", + "freeze", + "execution", + "measurements", + "evaluation", + "publication", + "stopConditions" + ], + "properties": { + "$schema": {"type": "string"}, + "schemaVersion": {"const": "1.0.0"}, + "protocolId": {"const": "aeg-v0.1.6-product-proof"}, + "status": {"const": "prepared-not-frozen"}, + "executionAuthorized": {"const": false}, + "evidenceBoundary": { + "type": "object", + "additionalProperties": false, + "required": ["verifiedRecordCount", "verifiedLibraryMutable", "challengeEvidence", "priorTransferEvidence", "claimsExcluded"], + "properties": { + "verifiedRecordCount": {"const": 2}, + "verifiedLibraryMutable": {"const": false}, + "challengeEvidence": {"const": "interaction-only"}, + "priorTransferEvidence": {"enum": ["neutral-or-negative"]}, + "claimsExcluded": { + "type": "array", + "minItems": 6, + "uniqueItems": true, + "items": {"enum": ["improved-success", "improved-speed", "improved-cost", "product-market-fit", "adoption", "generalization"]} + } + } + }, + "arms": { + "type": "array", + "minItems": 3, + "maxItems": 3, + "prefixItems": [ + {"$ref": "#/$defs/baselineArm"}, + {"$ref": "#/$defs/genericArm"}, + {"$ref": "#/$defs/aegArm"} + ], + "items": false + }, + "eligibility": { + "type": "object", + "additionalProperties": false, + "required": ["publicRepository", "explicitLicense", "excludedSources", "sourcePredatesTarget", "targetFrozenBeforeHumanPatch", "syntheticTargetForbidden", "candidatePoolFrozenBeforeRetrieval"], + "properties": { + "publicRepository": {"const": true}, + "explicitLicense": {"const": true}, + "excludedSources": { + "type": "array", + "minItems": 4, + "uniqueItems": true, + "items": {"enum": ["ebay", "private", "internal", "proprietary"]} + }, + "sourcePredatesTarget": {"const": true}, + "targetFrozenBeforeHumanPatch": {"const": true}, + "syntheticTargetForbidden": {"const": true}, + "candidatePoolFrozenBeforeRetrieval": {"const": true} + } + }, + "freeze": { + "type": "object", + "additionalProperties": false, + "required": ["requiredBeforeExecution", "target", "sourceExperience", "controls"], + "properties": { + "requiredBeforeExecution": {"const": true}, + "target": {"$ref": "#/$defs/unfrozenTarget"}, + "sourceExperience": {"$ref": "#/$defs/unfrozenSourceExperience"}, + "controls": {"$ref": "#/$defs/unfrozenControls"} + } + }, + "execution": { + "type": "object", + "additionalProperties": false, + "required": ["isolation", "freshContextPerArm", "sharedCaches", "armOrder", "maxAttemptsPerArm", "maxDurationSecondsPerArm", "humanPatchAvailableDuringArms", "executeInThisRelease"], + "properties": { + "isolation": {"const": "separate sanitized repositories and worktree directories with no shared Git metadata or writable caches"}, + "freshContextPerArm": {"const": true}, + "sharedCaches": {"const": false}, + "armOrder": {"type": "null"}, + "maxAttemptsPerArm": {"type": "integer", "minimum": 1}, + "maxDurationSecondsPerArm": {"type": "integer", "minimum": 1}, + "humanPatchAvailableDuringArms": {"const": false}, + "executeInThisRelease": {"const": false} + } + }, + "measurements": { + "type": "array", + "uniqueItems": true, + "minItems": 9, + "items": {"enum": ["success", "attempts", "commands", "tests", "nonCachedTokens", "durationMs", "filesInspected", "filesChanged", "patchHash"]} + }, + "evaluation": { + "type": "object", + "additionalProperties": false, + "required": ["focusedTestsRequired", "regressionTestsRequired", "sameOracleAcrossArms", "thresholdChangesAfterResults", "blindArmEvaluation", "successDefinition"], + "properties": { + "focusedTestsRequired": {"const": true}, + "regressionTestsRequired": {"const": true}, + "sameOracleAcrossArms": {"const": true}, + "thresholdChangesAfterResults": {"const": false}, + "blindArmEvaluation": {"const": true}, + "successDefinition": {"type": "string", "minLength": 20} + } + }, + "publication": { + "type": "object", + "additionalProperties": false, + "required": ["publishNegativeOutcomes", "publishAbstentions", "publishProtocolDeviations", "forbiddenArtifacts"], + "properties": { + "publishNegativeOutcomes": {"const": true}, + "publishAbstentions": {"const": true}, + "publishProtocolDeviations": {"const": true}, + "forbiddenArtifacts": { + "type": "array", + "minItems": 7, + "uniqueItems": true, + "items": {"enum": ["code", "prompts", "task-text", "raw-logs", "ratings", "receipts", "private-data"]} + } + } + }, + "stopConditions": { + "type": "array", + "minItems": 7, + "uniqueItems": true, + "items": {"type": "string", "minLength": 8} + } + }, + "$defs": { + "armBase": { + "type": "object", + "additionalProperties": false, + "required": ["id", "label", "additionalContext", "aegLibraryAccess", "targetInputs", "objectiveOracle"], + "properties": { + "id": {"type": "string"}, + "label": {"type": "string", "minLength": 3}, + "additionalContext": {"type": ["string", "null"]}, + "aegLibraryAccess": {"type": ["boolean", "string"]}, + "targetInputs": {"const": "identical-frozen"}, + "objectiveOracle": {"const": "identical-frozen"} + } + }, + "baselineArm": { + "allOf": [ + {"$ref": "#/$defs/armBase"}, + {"properties": {"id": {"const": "baseline"}, "additionalContext": {"type": "null"}, "aegLibraryAccess": {"const": false}}} + ] + }, + "genericArm": { + "allOf": [ + {"$ref": "#/$defs/armBase"}, + {"properties": {"id": {"const": "fixed-generic"}, "additionalContext": {"const": "generic-advice.txt"}, "aegLibraryAccess": {"const": false}}} + ] + }, + "aegArm": { + "allOf": [ + {"$ref": "#/$defs/armBase"}, + {"properties": {"id": {"const": "aeg-top-1"}, "additionalContext": {"const": "frozen-top-1-guarded-capsule"}, "aegLibraryAccess": {"const": "top-1-only"}}} + ] + }, + "unfrozenTarget": { + "type": "object", + "additionalProperties": false, + "required": ["repository", "license", "buggyCommitSha", "taskHash", "focusedTestHash", "regressionTestHash", "frozenAt"], + "properties": { + "repository": {"type": "null"}, + "license": {"type": "null"}, + "buggyCommitSha": {"type": "null"}, + "taskHash": {"type": "null"}, + "focusedTestHash": {"type": "null"}, + "regressionTestHash": {"type": "null"}, + "frozenAt": {"type": "null"} + } + }, + "unfrozenSourceExperience": { + "type": "object", + "additionalProperties": false, + "required": ["id", "recordedAt", "librarySha256", "retrievalScore", "retrievalThreshold", "capsuleSha256"], + "properties": { + "id": {"type": "null"}, + "recordedAt": {"type": "null"}, + "librarySha256": {"type": "null"}, + "retrievalScore": {"type": "null"}, + "retrievalThreshold": {"const": 0.05}, + "capsuleSha256": {"type": "null"} + } + }, + "unfrozenControls": { + "type": "object", + "additionalProperties": false, + "required": ["model", "settingsSha256", "promptTemplateSha256", "budgetSha256", "oracleSha256", "genericAdviceSha256"], + "properties": { + "model": {"type": "null"}, + "settingsSha256": {"type": "null"}, + "promptTemplateSha256": {"type": "null"}, + "budgetSha256": {"type": "null"}, + "oracleSha256": {"type": "null"}, + "genericAdviceSha256": {"const": "8d90bc397194fa3a6c146914e36860ffe887a77fb8de63ed9e5212099fdbd022"} + } + } + } +} diff --git a/experiments/v0.1.6-product-proof/result.schema.json b/experiments/v0.1.6-product-proof/result.schema.json new file mode 100644 index 0000000..b453184 --- /dev/null +++ b/experiments/v0.1.6-product-proof/result.schema.json @@ -0,0 +1,188 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://aeg.local/experiments/v0.1.6-product-proof/result.schema.json", + "title": "AEG v0.1.6 product-proof triad result", + "type": "object", + "additionalProperties": false, + "required": [ + "schemaVersion", + "protocolId", + "taskId", + "recordedAt", + "outcome", + "sourceExperience", + "retrieval", + "armResults", + "decision", + "limitations", + "privacy" + ], + "properties": { + "$schema": {"type": "string"}, + "schemaVersion": {"const": "1.0.0"}, + "protocolId": {"const": "aeg-v0.1.6-product-proof"}, + "taskId": {"type": "string", "pattern": "^[a-z0-9][a-z0-9.-]+$"}, + "recordedAt": {"type": "string", "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T"}, + "outcome": {"enum": ["completed", "retrieval-abstention", "protocol-deviation"]}, + "sourceExperience": { + "oneOf": [ + {"type": "null"}, + { + "type": "object", + "additionalProperties": false, + "required": ["id", "recordedAt", "predatesTarget", "librarySha256", "capsuleSha256"], + "properties": { + "id": {"type": "string", "pattern": "^trace-"}, + "recordedAt": {"type": "string"}, + "predatesTarget": {"const": true}, + "librarySha256": {"$ref": "#/$defs/sha256"}, + "capsuleSha256": {"$ref": "#/$defs/sha256"} + } + } + ] + }, + "retrieval": { + "type": "object", + "additionalProperties": false, + "required": ["decision", "score", "threshold", "matchedFields"], + "properties": { + "decision": {"enum": ["top-1", "abstain"]}, + "score": {"type": ["number", "null"], "minimum": 0}, + "threshold": {"const": 0.05}, + "matchedFields": { + "type": "array", + "uniqueItems": true, + "items": {"enum": ["task", "reuse.retrievalTags", "reuse.recommendedFor", "lessons", "subtasks.description", "subtasks.lessons"]} + } + } + }, + "armResults": { + "type": "array", + "items": {"$ref": "#/$defs/armResult"} + }, + "decision": {"enum": ["supported", "neutral", "negative", "abstention", "protocol-deviation"]}, + "limitations": { + "type": "array", + "minItems": 1, + "items": {"type": "string", "minLength": 10} + }, + "privacy": { + "type": "object", + "additionalProperties": false, + "required": ["containsCode", "containsPrompts", "containsTaskText", "containsRawLogs", "containsRatings", "containsReceipts", "containsPrivateData"], + "properties": { + "containsCode": {"const": false}, + "containsPrompts": {"const": false}, + "containsTaskText": {"const": false}, + "containsRawLogs": {"const": false}, + "containsRatings": {"const": false}, + "containsReceipts": {"const": false}, + "containsPrivateData": {"const": false} + } + } + }, + "allOf": [ + { + "if": {"properties": {"outcome": {"const": "completed"}}}, + "then": { + "properties": { + "sourceExperience": {"type": "object"}, + "retrieval": {"properties": {"decision": {"const": "top-1"}, "score": {"type": "number", "minimum": 0.05}}}, + "armResults": { + "minItems": 3, + "maxItems": 3, + "prefixItems": [ + {"allOf": [{"$ref": "#/$defs/armResult"}, {"properties": {"arm": {"const": "baseline"}}}]}, + {"allOf": [{"$ref": "#/$defs/armResult"}, {"properties": {"arm": {"const": "fixed-generic"}}}]}, + {"allOf": [{"$ref": "#/$defs/armResult"}, {"properties": {"arm": {"const": "aeg-top-1"}}}]} + ], + "items": false + }, + "decision": {"enum": ["supported", "neutral", "negative"]} + } + } + }, + { + "if": {"properties": {"outcome": {"const": "retrieval-abstention"}}}, + "then": { + "properties": { + "sourceExperience": {"type": "null"}, + "retrieval": {"properties": {"decision": {"const": "abstain"}, "score": {"type": ["number", "null"], "exclusiveMaximum": 0.05}}}, + "armResults": {"maxItems": 0}, + "decision": {"const": "abstention"} + } + } + }, + { + "if": {"properties": {"outcome": {"const": "protocol-deviation"}}}, + "then": {"properties": {"decision": {"const": "protocol-deviation"}}} + } + ], + "$defs": { + "sha256": {"type": "string", "pattern": "^[a-f0-9]{64}$"}, + "repoPath": {"type": "string", "pattern": "^(?!/)(?!.*(?:^|/)\\.\\.(?:/|$)).+$"}, + "checkResult": { + "type": "object", + "additionalProperties": false, + "required": ["commandSha256", "status"], + "properties": { + "commandSha256": {"$ref": "#/$defs/sha256"}, + "status": {"enum": ["passed", "failed", "not-run"]} + } + }, + "armResult": { + "type": "object", + "additionalProperties": false, + "required": ["arm", "success", "attempts", "commands", "tests", "nonCachedTokens", "durationMs", "files", "patchHash", "settings", "protocolDeviation"], + "properties": { + "arm": {"enum": ["baseline", "fixed-generic", "aeg-top-1"]}, + "success": {"type": "boolean"}, + "attempts": {"type": "integer", "minimum": 0}, + "commands": { + "type": "object", + "additionalProperties": false, + "required": ["completed", "sanitized"], + "properties": { + "completed": {"type": "integer", "minimum": 0}, + "sanitized": {"type": "array", "items": {"type": "string", "minLength": 1}} + } + }, + "tests": { + "type": "object", + "additionalProperties": false, + "required": ["executions", "focused", "regression"], + "properties": { + "executions": {"type": "integer", "minimum": 0}, + "focused": {"$ref": "#/$defs/checkResult"}, + "regression": {"$ref": "#/$defs/checkResult"} + } + }, + "nonCachedTokens": {"type": "integer", "minimum": 0}, + "durationMs": {"type": "integer", "minimum": 0}, + "files": { + "type": "object", + "additionalProperties": false, + "required": ["inspected", "changed"], + "properties": { + "inspected": {"type": "array", "uniqueItems": true, "items": {"$ref": "#/$defs/repoPath"}}, + "changed": {"type": "array", "uniqueItems": true, "items": {"$ref": "#/$defs/repoPath"}} + } + }, + "patchHash": {"$ref": "#/$defs/sha256"}, + "settings": { + "type": "object", + "additionalProperties": false, + "required": ["model", "settingsSha256", "promptTemplateSha256", "budgetSha256", "oracleSha256"], + "properties": { + "model": {"type": "string", "minLength": 1}, + "settingsSha256": {"$ref": "#/$defs/sha256"}, + "promptTemplateSha256": {"$ref": "#/$defs/sha256"}, + "budgetSha256": {"$ref": "#/$defs/sha256"}, + "oracleSha256": {"$ref": "#/$defs/sha256"} + } + }, + "protocolDeviation": {"type": ["string", "null"]} + } + } + } +} diff --git a/integrations/vscode/CHANGELOG.md b/integrations/vscode/CHANGELOG.md index a80e90a..318d3f8 100644 --- a/integrations/vscode/CHANGELOG.md +++ b/integrations/vscode/CHANGELOG.md @@ -1,5 +1,19 @@ # Changelog +## 0.1.6 + +- Make **AEG: Start with Verified Experience** the single dominant entry point + while preserving the previous command ID as a compatibility alias. +- Show the two-record/two-family coverage boundary and render abstention as a + correct, explained outcome with no fallback injection. +- Enforce evidence inspection, clipboard handoff, objective validation, and + query/experience-linked local feedback in order. +- Add exact normal-Chat paste instructions without using undocumented APIs. +- Move Playwright, Repair Lab, skill discovery, the bundled challenge, and + legacy commands into an Advanced sidebar section. +- Add a five-step first-install walkthrough, UX state-transition tests, and + non-executing three-arm protocol/result schema tests. + ## 0.1.5 - Rank verified experiences against top-level lessons, subtask descriptions, diff --git a/integrations/vscode/README.md b/integrations/vscode/README.md index 66349df..c606294 100644 --- a/integrations/vscode/README.md +++ b/integrations/vscode/README.md @@ -1,160 +1,96 @@ # Agent Experience Graph for VS Code -Retrieve verified debugging experience before your coding agent starts from -scratch. - -AEG v0.1.5 is a local-first developer preview: - -**Task or error → Explainable verified match → Guarded recovery capsule → Local rating** - -## What is new in v0.1.5 - -- Verified-experience retrieval now considers lessons and subtask evidence - without allowing long records to accumulate an unfair score advantage. -- Match explanations show the query terms that actually overlap, and nonzero - near-matches are disclosed when AEG abstains below its retrieval threshold. - -## Marketplace icon restored in v0.1.4 - -- Restore the original AEG Marketplace icon and package it explicitly. - -## Verified-experience workflow introduced in v0.1.3 - -- **AEG: Try a Verified Experience** searches the bundled, validated public - library. Candidate and malformed records are not eligible. -- Each card shows why it matched, the validated outcome, reusable lessons, - recommended use cases, constraints, limitations, and public provenance. -- **Copy capsule** produces concise context for a coding agent with an explicit - instruction to inspect the local code and run focused and regression tests. -- Helpful, partially helpful, irrelevant, and harmful ratings stay in - `.aeg/verified-experience-feedback.json`; task text and ratings are not - uploaded. -- **AEG: Open Verified Experience Challenge** opens a bundled synthetic - transfer task so a new user can see the full product loop immediately. -- **AEG: Run Public Repair Lab** launches isolated repairs of a real, - MIT-licensed FastAPI nested response-model bug by default. -- The baseline and assisted arms receive identical issue text, code, and tests; - only the assisted arm receives a compact retrieved recovery capsule. -- Repeated paired trials alternate execution order. Corrected telemetry captures - duration, completed commands, actual test runs, token usage, changed files, and - patches under `.aeg/repair-lab/`. -- The runner uses `codex exec --ephemeral --sandbox workspace-write`; it never - pushes code or contacts the upstream project. -- Verdicts require at least three trials and remain specific to the selected task. - -Run it from the AEG sidebar or command palette. The local `codex` executable -must be available on `PATH`. - -## First verified-experience retrieval - -1. Open a project in VS Code. -2. Select an error or describe a task with **AEG: Try a Verified Experience**. -3. Choose a match and inspect **Why this matched** and its limitations. -4. Copy the compact capsule into the coding-agent session before it begins the - repair. -5. Validate the result locally, then record whether retrieval was helpful. - -For an immediate demo, run **AEG: Open Verified Experience Challenge**. This is -a synthetic, non-identical transfer fixture. Its prior A/B pair produced the -same successful patch in both arms; retrieval changed neither repair path nor -outcome and increased token usage and wall time. It demonstrates the workflow, -not an AEG benefit claim. - -## Playwright diagnosis (from v0.1.1) - -- A dedicated **AEG Playwright** sidebar and status-bar entry point. -- Failure input from selected text, the latest Playwright artifact, the active file, a copied error, or a short description. -- Ten bundled Playwright recovery playbooks: - - timeouts - - unstable selectors - - authentication and session state - - network and API mocking - - flaky tests - - browser-specific failures - - test-data isolation - - CI-only failures - - trace and artifact diagnosis - - accessibility failures -- Local experience receipts using the minimum AEG structure: - - Intent - - Context - - Steps - - Skills - - Artifacts - - Failures - - Recovery - - Outcome - - Cost -- Explicit resolved/unresolved verification after a recovery attempt. -- Automatic detection of new text-based artifacts under `test-results`. - -## First diagnosis - -1. Open a project in VS Code. -2. Run a Playwright test and copy its error, or select an error/stack trace in the editor. -3. Click **AEG Playwright** in the Activity Bar or status bar. -4. Choose **Diagnose Playwright failure**. -5. Select a recommended playbook and try its recovery steps. -6. Re-run the test and mark the outcome **Test passed** or **Still failing**. - -AEG stores the receipt under: +AEG v0.1.6 is a local-first product-proof release with one primary workflow: -```text -.aeg/experiences/ +**Task or error → verified match or explicit abstention → evidence and limitations → guarded capsule handoff → observed validation → local feedback** + +AEG retrieves guidance. It does not automatically solve, send, or run the task. + +## Install a local build + +```bash +cd integrations/vscode +npm ci +npm test +npm run package +code --install-extension agent-experience-graph-0.1.6.vsix --force ``` -Use **AEG: Show Playwright Experiences** to inspect prior receipts. +Open any test workspace in a fresh VS Code window. The five-step walkthrough opens on first install and remains available as **AEG: Open Founder Proof Walkthrough**. -## Privacy +## Golden path + +1. Select error text or run **AEG: Start with Verified Experience** and enter a task. +2. AEG searches the bundled verified-only library locally. +3. If a result clears the fixed threshold, select it and inspect the exact matching phrases, weighted score, verified source outcome, provenance, constraints, and limitations. +4. Select **Copy capsule**. Open VS Code Chat from the Chat menu (macOS: Control+Command+I; Windows/Linux: Ctrl+Alt+I), paste into the chat input with the original task, and press Enter. For another coding agent, paste into its normal task or prompt input before it starts. +5. Run focused and regression checks. Record **Checks passed**, **Partially passed**, **Still failing**, or **Did not apply**. +6. Rate the selected experience **Helpful**, **Partially helpful**, **Irrelevant**, or **Harmful**. + +The panel keeps the original query and selected experience ID visible through validation and rating. It will not enable validation before handoff or rating before validation. + +## Honest coverage and abstention + +The public library contains exactly two verified records in two narrow task families: -Version 0.1.5 does not upload code, task descriptions, recovery capsules, logs, -artifacts, ratings, or experience receipts. +- agent evaluation and telemetry integrity; +- delegation and API contract repair. -- Receipts are local by default. -- Common authorization headers, passwords, tokens, API keys, and credential-bearing URLs are redacted from captured failure signatures. -- Artifact paths are recorded, but raw artifact contents are not written into receipts. -- Sharing is intentionally excluded until AEG has an explicit preview, consent, and redaction flow. +Retrieval is deterministic lexical ranking with a fixed 0.0500 threshold. **No relevant verified experience** is a correct outcome: AEG explains the best score or zero-score result, shows current coverage, and injects no candidate or generic fallback guidance. -Review a receipt before committing `.aeg/` to source control. Add `.aeg/` to `.gitignore` if the repository should not retain local experience data. +Verified means the recorded source outcome was objectively checked. It does not mean AEG retrieval improved correctness, success, speed, cost, or generalization. -## Existing skill commands +## Handoff API decision -The v0.1.0 local-skill workflow remains available: +VS Code documents `vscode.editorChat.start` for editor chat, but no stable extension API to open and prefill the normal Chat view. v0.1.6 therefore uses the supported clipboard API plus explicit paste-and-run instructions. It does not call an undocumented or private workbench command and never submits the capsule automatically. + +## Local feedback + +Feedback is appended to: + +```text +.aeg/verified-experience-feedback.json +``` + +Each row links a local proof-loop session, redacted query summary, selected experience ID and task, retrieval score, observed validation outcome, and rating. Review or ignore `.aeg/` before committing it. + +## Advanced capabilities + +The sidebar keeps prior capabilities under a collapsed **Advanced** section, and every existing command ID remains registered for backward compatibility: + +- bundled synthetic transfer challenge; +- Playwright artifact diagnosis, local receipts, and outcome marking; +- Public Repair Lab; +- workspace skill discovery, recommendation, rating, and local metrics; +- legacy getting-started content. + +The synthetic challenge demonstrates the interaction flow only. Its prior pair produced the same successful patch and repair path in both arms while assisted tokens and wall time were higher. The legacy Repair Lab and Playwright tools are not part of the default v0.1.6 path. + +## Privacy -- `AEG: Discover Workspace Skills` -- `AEG: Recommend Skill for Current Task` -- `AEG: Rate a Skill` -- `AEG: Show Skill Metrics` +AEG v0.1.6 does not upload code, task descriptions, prompts, recovery capsules, logs, artifacts, ratings, receipts, or private data. -These commands scan local `SKILL.md` and `capability.json` files. Skill metrics remain local in `.aeg/skill-metrics.json`. +- Search, ranking, clipboard handoff, and feedback are local. +- Common credentials are redacted from captured Playwright failure signatures. +- Raw Playwright artifact content is not written into receipts. +- There is no telemetry or sharing path in the verified-experience workflow. ## Development ```bash -npm install +npm ci npm test +npm run compile npm run package ``` -Open the extension directory in VS Code and press `F5` to launch an Extension Development Host. +`npm test` compiles TypeScript, runs the extension retrieval and UX state-transition tests, validates the product-proof protocol/result schemas, and verifies walkthrough assets. Packaging synchronizes the unchanged two-record verified library into the VSIX. ## Current limitations -- The verified public library contains only two records and supports no claim - of general coverage. -- Verified means the recorded outcome was objectively checked; it does not mean - AEG retrieval caused an improvement. -- Retrieval is deterministic lexical ranking, not embedding-based semantic - search. No match above the threshold means AEG abstains. -- The bundled transfer challenge is synthetic and is not cross-project - validation. Its prior controlled pair found no correctness or efficiency - benefit. -- Playbook ranking is deterministic keyword/signature matching, not semantic retrieval. -- AEG cannot read arbitrary integrated-terminal output; use a selection, clipboard, file, or Playwright artifact. -- Test outcome verification is user-confirmed in this release. -- Token counts are estimates based on captured text length. - -These constraints keep the first data loop understandable and auditable while -AEG recruits 5–10 seed users to test whether verified experience is useful on -their real debugging tasks. Please report a concrete retrieval outcome through -the repository issue tracker. +- The verified library has only two records and does not provide broad coverage. +- Retrieval is lexical, not embedding-based semantic search. +- The user confirms objective validation outcomes; AEG does not independently run or observe the checks. +- The normal Chat handoff is manual because no documented stable prefilled-Chat API is used. +- The bundled challenge demonstrates discoverability and interaction, not performance benefit. +- Prior transfer evidence is neutral or negative and supports no claim of improved success, speed, cost, PMF, adoption, or generalization. diff --git a/integrations/vscode/package-lock.json b/integrations/vscode/package-lock.json index 8e65cf7..cff1f4a 100644 --- a/integrations/vscode/package-lock.json +++ b/integrations/vscode/package-lock.json @@ -1,17 +1,18 @@ { "name": "agent-experience-graph", - "version": "0.1.5", + "version": "0.1.6", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "agent-experience-graph", - "version": "0.1.5", + "version": "0.1.6", "license": "MIT-0", "devDependencies": { "@types/node": "^20.14.0", "@types/vscode": "^1.90.0", "@vscode/vsce": "^3.6.2", + "ajv": "^8.20.0", "typescript": "^5.5.0" }, "engines": { diff --git a/integrations/vscode/package.json b/integrations/vscode/package.json index 33cea21..ecb5191 100644 --- a/integrations/vscode/package.json +++ b/integrations/vscode/package.json @@ -1,8 +1,8 @@ { "name": "agent-experience-graph", "displayName": "Agent Experience Graph", - "description": "Retrieve verified debugging experience before your coding agent starts from scratch, with local-only feedback and auditable evidence.", - "version": "0.1.5", + "description": "Search a small verified-experience library, inspect evidence and limitations, hand off guarded guidance, validate the outcome, and save local feedback.", + "version": "0.1.6", "publisher": "AgentExperienceGraph", "icon": "images/icon.png", "license": "MIT-0", @@ -24,20 +24,24 @@ "Other" ], "keywords": [ - "playwright", - "testing", "debugging", "agent", - "skills", - "experience" + "experience", + "verified", + "playwright", + "testing" ], "activationEvents": [ "onView:aeg.playwright", + "onCommand:aeg.startWithVerifiedExperience", "onCommand:aeg.tryVerifiedExperience", + "onCommand:aeg.showVerifiedCoverage", + "onCommand:aeg.openFounderWalkthrough", "onCommand:aeg.openVerifiedExperienceDemo", "onCommand:aeg.diagnosePlaywrightFailure", "onCommand:aeg.verifyLatestExperience", "onCommand:aeg.showExperiences", + "onCommand:aeg.openGettingStarted", "onCommand:aeg.runPublicRepairLab", "onCommand:aeg.discoverSkills", "onCommand:aeg.recommendSkill", @@ -48,60 +52,84 @@ "contributes": { "commands": [ { - "command": "aeg.tryVerifiedExperience", - "title": "AEG: Try a Verified Experience", + "command": "aeg.startWithVerifiedExperience", + "title": "AEG: Start with Verified Experience", "icon": "$(library)" }, + { + "command": "aeg.tryVerifiedExperience", + "title": "AEG: Try a Verified Experience (Compatibility Alias)" + }, + { + "command": "aeg.showVerifiedCoverage", + "title": "AEG: Show Verified Library Coverage", + "icon": "$(book)" + }, + { + "command": "aeg.openFounderWalkthrough", + "title": "AEG: Open Founder Proof Walkthrough", + "icon": "$(map)" + }, { "command": "aeg.openVerifiedExperienceDemo", - "title": "AEG: Open Verified Experience Challenge", + "title": "AEG (Advanced): Open Bundled Transfer Challenge", "icon": "$(lightbulb-autofix)" }, { "command": "aeg.diagnosePlaywrightFailure", - "title": "AEG: Diagnose Playwright Failure", + "title": "AEG (Advanced): Diagnose Playwright Failure", "icon": "$(debug-alt)" }, { "command": "aeg.verifyLatestExperience", - "title": "AEG: Mark Latest Diagnosis Outcome", + "title": "AEG (Advanced): Mark Latest Playwright Outcome", "icon": "$(pass)" }, { "command": "aeg.showExperiences", - "title": "AEG: Show Playwright Experiences", + "title": "AEG (Advanced): Show Playwright Experiences", "icon": "$(history)" }, { "command": "aeg.openGettingStarted", - "title": "AEG: Playwright Getting Started" + "title": "AEG (Advanced): Open Legacy Getting Started" }, { "command": "aeg.runPublicRepairLab", - "title": "AEG: Run Public Repair Lab", + "title": "AEG (Advanced): Run Public Repair Lab", "icon": "$(beaker)" }, { "command": "aeg.discoverSkills", - "title": "AEG: Discover Workspace Skills" + "title": "AEG (Advanced): Discover Workspace Skills" }, { "command": "aeg.recommendSkill", - "title": "AEG: Recommend Skill for Current Task" + "title": "AEG (Advanced): Recommend Skill for Current Task" }, { "command": "aeg.rateSkill", - "title": "AEG: Rate a Skill" + "title": "AEG (Advanced): Rate a Skill" }, { "command": "aeg.showSkillMetrics", - "title": "AEG: Show Skill Metrics" + "title": "AEG (Advanced): Show Skill Metrics" } ], "menus": { "commandPalette": [ { - "command": "aeg.tryVerifiedExperience" + "command": "aeg.startWithVerifiedExperience" + }, + { + "command": "aeg.tryVerifiedExperience", + "when": "false" + }, + { + "command": "aeg.showVerifiedCoverage" + }, + { + "command": "aeg.openFounderWalkthrough" }, { "command": "aeg.openVerifiedExperienceDemo" @@ -136,41 +164,21 @@ ], "editor/context": [ { - "command": "aeg.tryVerifiedExperience", + "command": "aeg.startWithVerifiedExperience", "when": "editorHasSelection", "group": "navigation@1" - }, - { - "command": "aeg.diagnosePlaywrightFailure", - "when": "editorHasSelection", - "group": "navigation@2" - }, - { - "command": "aeg.recommendSkill", - "when": "editorHasSelection", - "group": "navigation@3" } ], "view/title": [ { - "command": "aeg.tryVerifiedExperience", + "command": "aeg.startWithVerifiedExperience", "when": "view == aeg.playwright", "group": "navigation@1" }, { - "command": "aeg.diagnosePlaywrightFailure", + "command": "aeg.showVerifiedCoverage", "when": "view == aeg.playwright", "group": "navigation@2" - }, - { - "command": "aeg.showExperiences", - "when": "view == aeg.playwright", - "group": "navigation@3" - }, - { - "command": "aeg.runPublicRepairLab", - "when": "view == aeg.playwright", - "group": "navigation@4" } ] }, @@ -187,14 +195,79 @@ "aeg": [ { "id": "aeg.playwright", - "name": "Verified Experience & Playwright" + "name": "Verified Experience" } ] }, "viewsWelcome": [ { "view": "aeg.playwright", - "contents": "Retrieve verified debugging experience before your coding agent starts from scratch.\n[Try a verified experience](command:aeg.tryVerifiedExperience)\n[Open the bundled challenge](command:aeg.openVerifiedExperienceDemo)\n[Diagnose a Playwright failure](command:aeg.diagnosePlaywrightFailure)\n[Run public repair lab](command:aeg.runPublicRepairLab)\n[Open getting started](command:aeg.openGettingStarted)" + "contents": "Search two bundled verified records locally. AEG may correctly abstain.\n[Start with Verified Experience](command:aeg.startWithVerifiedExperience)\n[View coverage](command:aeg.showVerifiedCoverage)\n[Open guided walkthrough](command:aeg.openFounderWalkthrough)" + } + ], + "walkthroughs": [ + { + "id": "aegFounderProofLoop", + "title": "AEG verified-experience proof loop", + "description": "Complete one honest path from task or error to local outcome feedback.", + "steps": [ + { + "id": "start", + "title": "1. Start with a task or error", + "description": "Select an error first or enter a short task. Search stays local.\n[Start with Verified Experience](command:aeg.startWithVerifiedExperience)\n[Use the bundled guided task](command:aeg.openVerifiedExperienceDemo)", + "media": { + "markdown": "walkthrough/01-start.md" + }, + "completionEvents": [ + "onCommand:aeg.startWithVerifiedExperience", + "onCommand:aeg.openVerifiedExperienceDemo" + ] + }, + { + "id": "inspect", + "title": "2. Inspect the match or abstention", + "description": "Read the matching phrases, provenance, constraints, and limitations. A clear abstention is a correct outcome.", + "media": { + "markdown": "walkthrough/02-inspect.md" + }, + "completionEvents": [ + "onContext:aeg.proofLoopInspected || aeg.proofLoopAbstained" + ] + }, + { + "id": "handoff", + "title": "3. Copy and paste the guarded capsule", + "description": "Copy only an above-threshold capsule, then paste it into the normal chat input. AEG does not send or run it.", + "media": { + "markdown": "walkthrough/03-handoff.md" + }, + "completionEvents": [ + "onContext:aeg.proofLoopCopied" + ] + }, + { + "id": "validate", + "title": "4. Validate the outcome", + "description": "Run focused and regression checks, then record the observed result in the proof-loop panel.", + "media": { + "markdown": "walkthrough/04-validate.md" + }, + "completionEvents": [ + "onContext:aeg.proofLoopValidated" + ] + }, + { + "id": "rate", + "title": "5. Save local feedback", + "description": "Rate the selected experience as helpful, partially helpful, irrelevant, or harmful. The record stays in this workspace.", + "media": { + "markdown": "walkthrough/05-rate.md" + }, + "completionEvents": [ + "onContext:aeg.proofLoopRated" + ] + } + ] } ], "configuration": { @@ -221,7 +294,7 @@ "aeg.verifiedExperienceFeedbackFile": { "type": "string", "default": ".aeg/verified-experience-feedback.json", - "description": "Workspace-relative file for local usefulness ratings of retrieved verified experiences." + "description": "Workspace-relative file for local validation outcomes and usefulness ratings of retrieved verified experiences." }, "aeg.playwrightArtifactGlobs": { "type": "array", @@ -247,13 +320,14 @@ "sync:repair-lab": "node scripts/sync-repair-lab.js", "compile": "npm run sync:repair-lab && tsc -p ./", "watch": "tsc -watch -p ./", - "test": "npm run compile && node --test test/core.test.js", + "test": "npm run compile && node --test test/*.test.js", "package": "npx @vscode/vsce package" }, "devDependencies": { "@types/node": "^20.14.0", "@types/vscode": "^1.90.0", "@vscode/vsce": "^3.6.2", + "ajv": "^8.20.0", "typescript": "^5.5.0" } } diff --git a/integrations/vscode/src/extension.ts b/integrations/vscode/src/extension.ts index 8bc4e2e..664ef41 100644 --- a/integrations/vscode/src/extension.ts +++ b/integrations/vscode/src/extension.ts @@ -19,8 +19,16 @@ import { describeBelowThresholdMatch, generateRecoveryCapsule, loadVerifiedExperienceLibrary, - rankVerifiedExperiences + rankVerifiedExperiences, + summarizeVerifiedLibraryCoverage } from './verifiedExperience'; +import { + ProofLoopSession, + ValidationOutcome, + beginProofLoop, + proofLoopStep, + transitionProofLoop +} from './proofLoop'; const VERIFIED_EXPERIENCE_DEMO = 'Keepalive control fails after active stream ownership moved behind a protocol object; repair the public wrapper so it delegates through the protocol without using its stale socket field.'; @@ -55,9 +63,13 @@ class AegTreeItem extends vscode.TreeItem { label: string, description: string, icon: string, - command?: vscode.Command + command?: vscode.Command, + readonly children: AegTreeItem[] = [] ) { - super(label, vscode.TreeItemCollapsibleState.None); + super( + label, + children.length ? vscode.TreeItemCollapsibleState.Collapsed : vscode.TreeItemCollapsibleState.None + ); this.description = description; this.iconPath = new vscode.ThemeIcon(icon); this.command = command; @@ -68,6 +80,8 @@ class PlaywrightViewProvider implements vscode.TreeDataProvider { private readonly changeEmitter = new vscode.EventEmitter(); readonly onDidChangeTreeData = this.changeEmitter.event; + constructor(private readonly extensionUri: vscode.Uri) {} + refresh(): void { this.changeEmitter.fire(undefined); } @@ -76,18 +90,17 @@ class PlaywrightViewProvider implements vscode.TreeDataProvider { return element; } - async getChildren(): Promise { - const count = await countExperienceReceipts(); - return [ - new AegTreeItem( - 'Try a verified experience', - 'task → match → recovery capsule', - 'library', - {command: 'aeg.tryVerifiedExperience', title: 'Try verified experience'} - ), + async getChildren(element?: AegTreeItem): Promise { + if (element) return element.children; + const [receiptCount, library] = await Promise.all([ + countExperienceReceipts(), + readBundledVerifiedLibrary(this.extensionUri) + ]); + const coverage = summarizeVerifiedLibraryCoverage(library.experiences); + const advanced = [ new AegTreeItem( - 'Open the transfer challenge', - 'bundled zero-cold-start demo', + 'Bundled transfer challenge', + 'synthetic workflow demonstration', 'lightbulb-autofix', {command: 'aeg.openVerifiedExperienceDemo', title: 'Open challenge'} ), @@ -98,28 +111,85 @@ class PlaywrightViewProvider implements vscode.TreeDataProvider { {command: 'aeg.diagnosePlaywrightFailure', title: 'Diagnose'} ), new AegTreeItem( - 'Mark latest outcome', + 'Mark latest Playwright outcome', 'resolved or unresolved', 'pass', {command: 'aeg.verifyLatestExperience', title: 'Verify'} ), new AegTreeItem( - 'Local experience receipts', - `${count} recorded`, + 'Local Playwright receipts', + `${receiptCount} recorded`, 'history', {command: 'aeg.showExperiences', title: 'Show experiences'} ), new AegTreeItem( - 'Run public repair lab', - 'baseline vs AEG-assisted', + 'Run public Repair Lab', + 'legacy controlled experiment runner', 'beaker', {command: 'aeg.runPublicRepairLab', title: 'Run repair lab'} ), new AegTreeItem( - 'Privacy', - 'local only · no upload', - 'shield', - {command: 'aeg.openGettingStarted', title: 'Getting started'} + 'Discover workspace skills', + 'legacy local skill discovery', + 'search', + {command: 'aeg.discoverSkills', title: 'Discover skills'} + ), + new AegTreeItem( + 'Recommend a workspace skill', + 'legacy local skill ranking', + 'wand', + {command: 'aeg.recommendSkill', title: 'Recommend skill'} + ), + new AegTreeItem( + 'Rate a workspace skill', + 'legacy local skill feedback', + 'star-empty', + {command: 'aeg.rateSkill', title: 'Rate skill'} + ), + new AegTreeItem( + 'Show skill metrics', + 'legacy local metrics', + 'graph', + {command: 'aeg.showSkillMetrics', title: 'Show metrics'} + ), + new AegTreeItem( + 'Legacy verified-experience alias', + 'backward-compatible command ID', + 'debug-step-back', + {command: 'aeg.tryVerifiedExperience', title: 'Compatibility alias'} + ), + new AegTreeItem( + 'Legacy getting started', + 'Playwright and Repair Lab reference', + 'file-text', + {command: 'aeg.openGettingStarted', title: 'Legacy getting started'} + ) + ]; + return [ + new AegTreeItem( + 'Start with Verified Experience', + 'search → inspect → hand off → validate → rate', + 'library', + {command: 'aeg.startWithVerifiedExperience', title: 'Start with Verified Experience'} + ), + new AegTreeItem( + 'Verified library coverage', + `${coverage.verifiedRecordCount} records · ${coverage.families.length} task families`, + 'book', + {command: 'aeg.showVerifiedCoverage', title: 'Show verified coverage'} + ), + new AegTreeItem( + 'Guided walkthrough', + 'complete the proof loop without the README', + 'map', + {command: 'aeg.openFounderWalkthrough', title: 'Open walkthrough'} + ), + new AegTreeItem( + 'Advanced', + 'Playwright · Repair Lab · skills · legacy', + 'tools', + undefined, + advanced ) ]; } @@ -127,12 +197,12 @@ class PlaywrightViewProvider implements vscode.TreeDataProvider { export function activate(context: vscode.ExtensionContext): void { const status = vscode.window.createStatusBarItem(vscode.StatusBarAlignment.Left, 100); - status.text = '$(library) AEG Experience'; - status.command = 'aeg.tryVerifiedExperience'; - status.tooltip = 'Retrieve verified debugging experience before starting from scratch'; + status.text = '$(library) AEG Verified Experience'; + status.command = 'aeg.startWithVerifiedExperience'; + status.tooltip = 'Search the two-record verified library; AEG may correctly abstain'; status.show(); - const viewProvider = new PlaywrightViewProvider(); + const viewProvider = new PlaywrightViewProvider(context.extensionUri); const tree = vscode.window.createTreeView('aeg.playwright', {treeDataProvider: viewProvider}); const notifiedArtifacts = new Set(); const watcher = vscode.workspace.createFileSystemWatcher('**/test-results/**/*'); @@ -158,6 +228,10 @@ export function activate(context: vscode.ExtensionContext): void { status, tree, watcher, + vscode.commands.registerCommand( + 'aeg.startWithVerifiedExperience', + () => tryVerifiedExperience(context.extensionUri) + ), vscode.commands.registerCommand( 'aeg.tryVerifiedExperience', () => tryVerifiedExperience(context.extensionUri) @@ -166,6 +240,18 @@ export function activate(context: vscode.ExtensionContext): void { 'aeg.openVerifiedExperienceDemo', () => tryVerifiedExperience(context.extensionUri, VERIFIED_EXPERIENCE_DEMO) ), + vscode.commands.registerCommand( + 'aeg.showVerifiedCoverage', + () => showVerifiedCoverage(context.extensionUri) + ), + vscode.commands.registerCommand( + 'aeg.openFounderWalkthrough', + () => vscode.commands.executeCommand( + 'workbench.action.openWalkthrough', + 'AgentExperienceGraph.agent-experience-graph#aegFounderProofLoop', + false + ) + ), vscode.commands.registerCommand( 'aeg.diagnosePlaywrightFailure', (uri?: vscode.Uri) => diagnosePlaywrightFailure(viewProvider, uri) @@ -194,37 +280,28 @@ async function tryVerifiedExperience(extensionUri: vscode.Uri, presetTask?: stri ? redactSensitiveText(editor.document.getText(editor.selection), 1_000) : ''; const task = presetTask ?? await vscode.window.showInputBox({ - title: 'AEG: Try a Verified Experience', - prompt: 'Describe the task, error, or issue. AEG ranks only bundled verified records; it does not upload this text.', + title: 'AEG: Start with Verified Experience', + prompt: 'Enter a task or use the selected error text. AEG searches two bundled verified records locally and may correctly abstain.', value: selectedText, placeHolder: 'A protocol wrapper still uses a stale socket after stream ownership moved' }); if (!task?.trim()) return; - const libraryUri = vscode.Uri.joinPath(extensionUri, 'verified-experiences', 'verified.json'); - let raw: string; - try { - raw = Buffer.from(await vscode.workspace.fs.readFile(libraryUri)).toString('utf8'); - } catch { - void vscode.window.showErrorMessage('The bundled verified-experience library is missing. Reinstall AEG v0.1.5.'); + let session = beginProofLoop(task, `proof-${Date.now()}`); + const library = await readBundledVerifiedLibrary(extensionUri); + if (!library.experiences.length && library.malformed.some(item => item.includes('missing'))) { + void vscode.window.showErrorMessage('The bundled verified-experience library is missing. Reinstall AEG v0.1.6.'); return; } - - const library = loadVerifiedExperienceLibrary(raw); if (library.malformed.length) { void vscode.window.showWarningMessage(`AEG excluded ${library.malformed.length} malformed verified-experience record(s).`); } const matches = rankVerifiedExperiences(task, library.experiences); if (!matches.length) { const nearMatch = rankVerifiedExperiences(task, library.experiences, Number.EPSILON, 1)[0]; - if (nearMatch) { - void vscode.window.showInformationMessage(describeBelowThresholdMatch(nearMatch)); - } else { - void vscode.window.showInformationMessage( - `AEG found no lexical match (score 0; threshold ${VERIFIED_EXPERIENCE_RETRIEVAL_THRESHOLD.toFixed(4)}). ` - + 'No candidate or fallback guidance was injected.' - ); - } + session = transitionProofLoop(session, {type: 'abstain'}); + await vscode.commands.executeCommand('setContext', 'aeg.proofLoopAbstained', true); + showVerifiedExperienceAbstention(session, library.experiences, nearMatch); return; } @@ -238,46 +315,108 @@ async function tryVerifiedExperience(extensionUri: vscode.Uri, presetTask?: stri match })), { - title: 'Verified experience matches', - placeHolder: 'Choose a verified record to inspect before your coding agent starts', + title: `Step 2 of 5 · ${library.experiences.length} verified records searched`, + placeHolder: 'Select a verified record to inspect; this does not claim it will solve the task', matchOnDescription: true, matchOnDetail: true } ); if (!picked) return; - await showVerifiedExperiencePanel(task, picked.match); + session = transitionProofLoop(session, {type: 'match', match: picked.match}); + session = transitionProofLoop(session, {type: 'inspect'}); + await vscode.commands.executeCommand('setContext', 'aeg.proofLoopInspected', true); + await showVerifiedExperiencePanel(session, picked.match); +} + +async function readBundledVerifiedLibrary(extensionUri: vscode.Uri) { + const libraryUri = vscode.Uri.joinPath(extensionUri, 'verified-experiences', 'verified.json'); + try { + const raw = Buffer.from(await vscode.workspace.fs.readFile(libraryUri)).toString('utf8'); + return loadVerifiedExperienceLibrary(raw); + } catch { + return {experiences: [], malformed: ['bundled verified-experience library is missing']}; + } } -async function showVerifiedExperiencePanel(task: string, match: VerifiedExperienceMatch): Promise { +async function showVerifiedExperiencePanel( + initialSession: ProofLoopSession, + match: VerifiedExperienceMatch +): Promise { const panel = vscode.window.createWebviewPanel( 'aegVerifiedExperience', - 'AEG: Verified Experience', + 'AEG: Verified Experience Proof Loop', vscode.ViewColumn.Beside, {enableScripts: true} ); + let session = initialSession; + let feedbackPending = false; const capsule = generateRecoveryCapsule(match); - panel.webview.html = verifiedExperienceHtml(match, capsule, panel.webview); + panel.webview.html = verifiedExperienceHtml(session, match, capsule, panel.webview); panel.webview.onDidReceiveMessage(async message => { if (message?.command === 'copy') { + if (session.stage !== 'inspected') return; + const copied = transitionProofLoop(session, {type: 'copy'}); await vscode.env.clipboard.writeText(capsule); - void vscode.window.showInformationMessage('Verified-experience recovery capsule copied. Validate it against the local repository before applying changes.'); + session = copied; + await vscode.commands.executeCommand('setContext', 'aeg.proofLoopCopied', true); + await postProofLoopState(panel, session); + void vscode.window.showInformationMessage( + 'Capsule copied. Open VS Code Chat, paste it into the chat input with your task, press Enter, then validate the result with focused and regression checks.' + ); + return; + } + if (message?.command === 'validate' && isValidationOutcome(message.outcome)) { + if (session.stage !== 'copied') return; + session = transitionProofLoop(session, {type: 'validate', outcome: message.outcome}); + await vscode.commands.executeCommand('setContext', 'aeg.proofLoopValidated', true); + await postProofLoopState(panel, session); return; } if (message?.command === 'rate' && isExperienceRating(message.rating)) { - const saved = await writeVerifiedExperienceFeedback(task, match, message.rating); - if (saved) void vscode.window.showInformationMessage(`AEG recorded “${message.rating}” feedback locally.`); + if (session.stage !== 'validated' || feedbackPending) return; + const rated = transitionProofLoop(session, {type: 'rate', rating: message.rating}); + feedbackPending = true; + try { + const saved = await writeVerifiedExperienceFeedback(rated, match); + if (saved) { + session = rated; + await vscode.commands.executeCommand('setContext', 'aeg.proofLoopRated', true); + await postProofLoopState(panel, session); + void vscode.window.showInformationMessage( + `AEG recorded “${message.rating}” feedback for ${match.experience.id} locally.` + ); + } + } finally { + feedbackPending = false; + } } }); } +async function postProofLoopState( + panel: vscode.WebviewPanel, + session: ProofLoopSession +): Promise { + await panel.webview.postMessage({ + type: 'proof-loop-state', + stage: session.stage, + step: proofLoopStep(session.stage), + validationOutcome: session.validationOutcome, + rating: session.rating + }); +} + function isExperienceRating(value: unknown): value is ExperienceRating { return value === 'helpful' || value === 'partially-helpful' || value === 'irrelevant' || value === 'harmful'; } +function isValidationOutcome(value: unknown): value is ValidationOutcome { + return value === 'passed' || value === 'partially-passed' || value === 'failed' || value === 'not-applied'; +} + async function writeVerifiedExperienceFeedback( - task: string, - match: VerifiedExperienceMatch, - rating: ExperienceRating + session: ProofLoopSession, + match: VerifiedExperienceMatch ): Promise { const root = workspaceRoot(); if (!root) { @@ -297,12 +436,16 @@ async function writeVerifiedExperienceFeedback( } catch { // A missing file is the expected first-use state. } + if (!session.rating || !session.validationOutcome) return false; const feedback: ExperienceFeedback = { - schemaVersion: '1.0.0', + schemaVersion: '1.1.0', recordedAt: new Date().toISOString(), + proofLoopSessionId: session.id, experienceId: match.experience.id, - taskSummary: redactSensitiveText(task, 500), - rating, + experienceTask: match.experience.task, + taskSummary: redactSensitiveText(session.query, 500), + rating: session.rating, + validationOutcome: session.validationOutcome, retrievalScore: match.score, localOnly: true }; @@ -314,6 +457,7 @@ async function writeVerifiedExperienceFeedback( } function verifiedExperienceHtml( + session: ProofLoopSession, match: VerifiedExperienceMatch, capsule: string, webview: vscode.Webview @@ -332,27 +476,90 @@ body{padding:28px;max-width:920px;margin:auto;font:14px/1.55 var(--vscode-font-f .badge{display:inline-block;padding:4px 9px;border-radius:999px;background:var(--vscode-badge-background);color:var(--vscode-badge-foreground)} .card,.guardrail{margin:18px 0;padding:18px;border:1px solid var(--vscode-widget-border);border-radius:10px;background:var(--vscode-editor-background)} .guardrail{border-left:4px solid var(--vscode-editorWarning-foreground)} +.query{border-left:4px solid var(--vscode-focusBorder)} +.pending{opacity:.6}.complete{border-left:4px solid var(--vscode-testing-iconPassed)} .grid{display:grid;grid-template-columns:1fr 1fr;gap:16px} h1{line-height:1.15} h2{margin-top:26px} li{margin:6px 0} table{width:100%;border-collapse:collapse}th,td{text-align:left;padding:7px;border-bottom:1px solid var(--vscode-widget-border)} pre{white-space:pre-wrap;padding:14px;background:var(--vscode-textCodeBlock-background);overflow:auto} button{margin:6px 8px 0 0;padding:8px 12px;color:var(--vscode-button-foreground);background:var(--vscode-button-background);border:0;border-radius:4px;cursor:pointer} button.secondary{color:var(--vscode-button-secondaryForeground);background:var(--vscode-button-secondaryBackground)} +button:disabled{opacity:.45;cursor:not-allowed}.status{font-weight:600} @media(max-width:650px){.grid{grid-template-columns:1fr}} +Step 2 of 5 · verified match +

Inspect evidence before handoff

+

Original query

${escapeHtml(session.query)}

Session ${escapeHtml(session.id)} · stays local

${Math.round(match.score * 100)}% weighted match · verification ${escapeHtml(experience.verification.status)} -

${escapeHtml(experience.task)}

+

${escapeHtml(experience.task)}

${escapeHtml(experience.id)}

Guidance, not a guaranteed answer. Inspect the local code, reproduce the failure, and validate any repair with focused and regression tests.

Reusable lessons

    ${list(experience.lessons)}

Recommended use cases

    ${list(experience.reuse.recommendedFor)}

Constraints

    ${list(experience.constraints)}

Limitations

    ${list(experience.limitations)}

Why this matched

${evidence}
FieldTask phraseExperience phraseLexicalWeighted

Provenance and outcome

Outcome: ${escapeHtml(experience.outcome)}. Public source: ${escapeHtml(source.repository)} · ${escapeHtml(source.license)} · ${escapeHtml(source.benchmark)}. Experiment artifact: ${escapeHtml(experience.provenance.experimentEvidence.artifact)}.

-

Compact recovery capsule

${escapeHtml(capsule)}

-

Was this useful?

-

Ratings are written only to this workspace under .aeg/. AEG v0.1.5 does not upload task text, code, capsules, or ratings.

- +

Step 3 · Guarded capsule handoff

${escapeHtml(capsule)}

+

Step 4 · Validate the outcome

For the original query above, run a focused check and relevant regression checks. What objective result did you observe?

+

Step 5 · Rate this retrieval

Rate ${escapeHtml(experience.id)} for the original query after recording the validation outcome.

+

Feedback is written only to this workspace under .aeg/verified-experience-feedback.json. AEG v0.1.6 does not upload task text, code, prompts, logs, capsules, ratings, receipts, or private data.

+ `; } +function showVerifiedExperienceAbstention( + session: ProofLoopSession, + experiences: ReturnType['experiences'], + nearMatch?: VerifiedExperienceMatch +): void { + const panel = vscode.window.createWebviewPanel( + 'aegVerifiedExperienceAbstention', + 'AEG: No Relevant Verified Experience', + vscode.ViewColumn.Beside, + {enableScripts: true} + ); + const coverage = summarizeVerifiedLibraryCoverage(experiences); + const explanation = nearMatch + ? describeBelowThresholdMatch(nearMatch) + : `AEG found no lexical match (score 0; threshold ${VERIFIED_EXPERIENCE_RETRIEVAL_THRESHOLD.toFixed(4)}). No candidate or fallback guidance was injected.`; + const nonce = `${Date.now()}`; + panel.webview.html = `

Step 2 of 5 · retrieval outcome

No relevant verified experience

This is a correct product outcome, not an error. AEG did not force a weak match or inject generic fallback guidance.

Original query

${escapeHtml(session.query)}

Why AEG abstained

${escapeHtml(explanation)}

Current coverage

${coverage.verifiedRecordCount} verified records across ${coverage.families.length} task families:

    ${coverage.families.map(family => `
  • ${escapeHtml(family.label)} — ${escapeHtml(family.description)}
    • `).join('')}

Tasks outside those narrow families will often and correctly abstain.

No code, task text, prompt, log, rating, receipt, or private data was uploaded.

`; + panel.webview.onDidReceiveMessage(async message => { + if (message?.command === 'retry') { + panel.dispose(); + await vscode.commands.executeCommand('aeg.startWithVerifiedExperience'); + } else if (message?.command === 'coverage') { + await vscode.commands.executeCommand('aeg.showVerifiedCoverage'); + } + }); +} + +async function showVerifiedCoverage(extensionUri: vscode.Uri): Promise { + const library = await readBundledVerifiedLibrary(extensionUri); + const coverage = summarizeVerifiedLibraryCoverage(library.experiences); + const families = coverage.families.map(family => + `- **${family.label}** (${family.recordCount} record): ${family.description}` + ).join('\n'); + const document = await vscode.workspace.openTextDocument({ + language: 'markdown', + content: `# AEG verified-library coverage\n\n` + + `**${coverage.verifiedRecordCount} verified records across ${coverage.families.length} narrow task families.**\n\n${families}\n\n` + + `AEG uses deterministic lexical ranking with a fixed ${VERIFIED_EXPERIENCE_RETRIEVAL_THRESHOLD.toFixed(4)} threshold. ` + + `Below-threshold and zero-score results are shown as **No relevant verified experience**. ` + + `That abstention is expected and injects no candidate or generic fallback guidance.\n\n` + + `Verified means the source outcome was objectively checked. It does not mean retrieval improves success, speed, cost, or generalization.\n` + }); + await vscode.window.showTextDocument(document, {preview: true}); +} + async function diagnosePlaywrightFailure( viewProvider: PlaywrightViewProvider, preferredArtifact?: vscode.Uri @@ -656,7 +863,7 @@ function recoveryHtml(

Verify the outcome

After trying the playbook and re-running the test, record the objective result.

-

Local only: AEG v0.1.5 does not upload code, logs, or experience receipts.

+

Local only: AEG v0.1.6 does not upload code, logs, or experience receipts.