From 2734f233bda1106c9b24360f15f7f14c3e7f14a5 Mon Sep 17 00:00:00 2001 From: Note Flow AI Date: Tue, 15 Sep 2026 23:23:39 +0800 Subject: [PATCH] Prevent stale evidence links from restoring changed audits; release 0.10.2 --- CHANGELOG.md | 6 +++++ README.md | 5 +++- README.zh-CN.md | 4 +++- docs/explorer.md | 6 +++++ package-lock.json | 4 ++-- package.json | 2 +- pyproject.toml | 2 +- scripts/check_site.cjs | 52 ++++++++++++++++++++++++++++++++++++++++- site/app.js | 45 ++++++++++++++++++++++++++++------- site/index.html | 1 + src/evalarc/__init__.py | 2 +- 11 files changed, 113 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 24a1ae4..c59e0dd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## 0.10.2 — 2026-09-15 + +- Bind evidence explorer links to SHA-256 of the actual loaded audit bytes. A changed recording with the same case coordinates no longer reports the original evidence as restored. Display the loaded fingerprint and distinguish content identity from authorship. +- Keep legacy links readable with an explicit missing-identity notice; reject duplicate parameters and preserve report/download access when browser hashing is unavailable. Test changed bytes under an unchanged manifest, fresh links, retry, mobile layouts and clipboard fallback. +- Task contracts, metric calculations and historical audit records are unchanged. + ## 0.10.1 — 2026-09-15 - Add a reduced-suite regression that removes `cas-type-sensitivity` from `durable-kv` at seed 17: `boolean-equals-one` survives, the recomputed audit reports 7 of 8 at 0.875, and the weakest margin is 0. The published 0.10.0 wheel already produces these values and includes surviving valid controls in the weakest margin. This release adds the regression and clarifies the explanation; it does not change the metric calculation. diff --git a/README.md b/README.md index 90c19d1..5de78f1 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,10 @@ lets you compare revisions side by side, switch between correct and faulty implementations, and step through the tool call that changed the state. It replays the committed Docker audits without a model API or installation. Share the exact case and trace step with **Copy evidence link**, return from -details to the case list, and retry failed sections independently. +details to the case list, and retry failed sections independently. New links include +SHA-256 of the loaded audit bytes: changed evidence is flagged before restoring a +view, while legacy links disclose that the original audit identity is unknown. +The fingerprint identifies content, not its author. [Explorer guide](docs/explorer.md). [![EvalArc v0.3: score rises from 90% to 93.75% while a check regresses](docs/assets/regression-lab.png)](https://huggingface.co/spaces/glayguo/evalarc) diff --git a/README.zh-CN.md b/README.zh-CN.md index 535762d..65fa182 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -22,7 +22,9 @@ EvalArc 关注智能体实际完成的结果，以及支撑评分结论的证据 可以并排比较两个版本，查看一处退步、两处改进，再逐步检查工具调用和状态变化。 页面读取仓库保存的 Docker 审计记录，无需安装，也不调用模型。 使用 **Copy evidence link** 分享具体案例和步骤；打开详情后可返回原案例， -某一分区加载失败时可单独重试。[交互使用说明](docs/explorer.md)。 +某一分区加载失败时可单独重试。新链接携带实际加载审计文件的 SHA-256， +记录变化时会先提示并停止恢复；旧链接明确说明未保存原始审计身份。 +指纹标识内容，不认证作者。[交互使用说明](docs/explorer.md)。 [![EvalArc v0.3：分数上升，一项检查却退步](docs/assets/regression-lab.png)](https://huggingface.co/spaces/glayguo/evalarc) diff --git a/docs/explorer.md b/docs/explorer.md index fe65b60..c644e4c 100644 --- a/docs/explorer.md +++ b/docs/explorer.md @@ -23,3 +23,9 @@ The browser checks exercise desktop and mobile layouts, clipboard denial, reload/history restoration, keyboard focus, partial loading and retries. They do not constitute a full accessibility certification or a browser support matrix. The original casebook and audit bytes retain their recorded provenance. + +## Evidence identity in shared links + +Version 2 links include SHA-256 of the exact audit response bytes, together with the task, control, case, seed and trace step. The explorer hashes the fetched audit itself rather than trusting a potentially stale site manifest. If it differs, the linked view is not restored; choose a current case or open the saved original audit. Version 1 links show an explicit legacy notice because they cannot identify the original audit. Duplicate parameters and unsupported versions are rejected. + +This is content identity, not an author signature or a substitute for offline evidence verification. HTTPS (or localhost) provides browser hashing. If it is unavailable, reports and downloads remain accessible while evidence-bound sharing is disabled. diff --git a/package-lock.json b/package-lock.json index 2d2089d..d1a9f06 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "evalarc-evidence-site", - "version": "0.10.0", + "version": "0.10.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "evalarc-evidence-site", - "version": "0.10.0", + "version": "0.10.2", "devDependencies": { "playwright": "1.63.0" } diff --git a/package.json b/package.json index d21c9d5..3151bc7 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "evalarc-evidence-site", - "version": "0.10.0", + "version": "0.10.2", "private": true, "description": "Browser checks for the static EvalArc evidence explorer", "scripts": { diff --git a/pyproject.toml b/pyproject.toml index 8792d39..6d13990 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "evalarc" -version = "0.10.1" +version = "0.10.2" description = "Auditable task environments and evaluations for coding and tool-using agents." readme = "README.md" requires-python = ">=3.11" diff --git a/scripts/check_site.cjs b/scripts/check_site.cjs index 1540ec8..b835d17 100644 --- a/scripts/check_site.cjs +++ b/scripts/check_site.cjs @@ -141,6 +141,8 @@ async function main() { await app.locator("#share-case").click(); const shared = await app.locator("#share-url").inputValue(); const sharedParams = new URLSearchParams(new URL(shared).hash.slice(1)); + assert.equal(sharedParams.get("v"), "2"); + assert.equal(sharedParams.get("audit"), manifest.files["support/audit.json"]); assert.equal(sharedParams.get("case"), "retry-after-commit"); assert.equal(sharedParams.get("step"), "3"); assert.equal(await app.locator("body").evaluate(() => document.activeElement.id), "share-url"); @@ -292,7 +294,7 @@ async function main() { assert.equal(await page.locator("#changed-cases button").count(), 3); assert.equal(await page.locator("#repeat-attempts a").count(), 3); assert.equal(await page.locator("#cases button").count(), 15); - assert.match(await page.locator("#share-status").innerText(), /Shared evidence restored/); + assert.match(await page.locator("#share-status").innerText(), /Legacy link.*original audit identity was not recorded/); assert.equal(await page.locator("body").evaluate(() => document.documentElement.scrollWidth > innerWidth), false); assert.deepEqual(errors, []); failures.add("skill-impact/lab.json"); @@ -304,6 +306,54 @@ async function main() { assert.equal(await page.locator(".trial").count(),9); results.push({independentSectionRetry:true, partialTaskPack:true, pendingLinkRestored:true, researchRetry:true, errors}); await page.close(); + const identityPage = await browser.newPage({viewport:{width:390,height:1000}}); + try { + const auditBytes = fs.readFileSync(path.join(root, "support/audit.json")); + const digest = require("node:crypto").createHash("sha256").update(auditBytes).digest("hex"); + const link = base + `#v=2&audit=${digest}&pack=support&control=new-key-on-retry&case=retry-after-commit&seed=17&step=3`; + await identityPage.goto(link); + await identityPage.locator("#share-status").filter({hasText:"Shared evidence restored"}).waitFor(); + assert.equal(await identityPage.locator("#trace-step").inputValue(), "3"); + const changed = JSON.parse(auditBytes); + changed.reference.created_at = "2026-09-15T00:00:00+00:00"; + const changedBytes = Buffer.from(JSON.stringify(changed) + "\n"); + const changedDigest = require("node:crypto").createHash("sha256").update(changedBytes).digest("hex"); + assert.notEqual(changedDigest, digest); + // Same task, controls, cases and step coordinates, but different saved + // audit bytes. The manifest deliberately stays old, as in a mixed deploy. + await identityPage.route("**/support/audit.json", route => + route.fulfill({status:200, contentType:"application/json", body:changedBytes})); + await identityPage.reload(); + await identityPage.locator("#share-status").filter({hasText:"Evidence changed"}).waitFor(); + assert.equal(await identityPage.locator("#trace-step").inputValue(), "0"); + assert.match(await identityPage.locator("#share-status").innerText(), /linked view was not restored/); + assert.equal(JSON.parse(await identityPage.locator("#provenance").textContent()).audit_sha256, changedDigest); + await identityPage.locator("#next-step").click(); + const fresh = identityPage.url(); + assert.equal(new URLSearchParams(new URL(fresh).hash.slice(1)).get("audit"), changedDigest); + await identityPage.reload(); + await identityPage.locator("#share-status").filter({hasText:"Shared evidence restored"}).waitFor(); + assert.equal(await identityPage.locator("#trace-step").inputValue(), "1"); + await identityPage.evaluate(hash => { location.hash = hash; }, + new URL(fresh).hash + "&audit=" + digest); + await identityPage.locator("#share-status").filter({hasText:"not supported"}).waitFor(); + assert.equal(await identityPage.locator("#trace-step").inputValue(), "1"); + await identityPage.goto(base + "#v=1&pack=support&control=new-key-on-retry&case=retry-after-commit&seed=17&step=3"); + await identityPage.locator("#share-status").filter({hasText:"Legacy link"}).waitFor(); + assert.equal(await identityPage.locator("#trace-step").inputValue(), "3"); + await identityPage.addInitScript(() => { + Object.defineProperty(crypto, "subtle", {value:undefined, configurable:true}); + }); + await identityPage.goto(link); + await identityPage.reload(); + await identityPage.locator("#workspace").waitFor({state:"visible"}); + assert(await identityPage.locator("#share-case").isDisabled()); + assert.match(await identityPage.locator("#audit-identity").innerText(), /fingerprint unavailable/); + assert.match(await identityPage.locator("#share-status").innerText(), /linked view was not restored/); + assert(await identityPage.locator("#suite-workspace").isVisible()); + results.push({auditByteIdentity:true, changedAuditRejected:true, freshLinkRestored:true, + duplicateHashRejected:true, legacyLinkDisclosed:true, missingCryptoRecovery:true}); + } finally { await identityPage.close(); } } console.log(JSON.stringify({url:base, checks:results}, null, 2)); } finally { diff --git a/site/app.js b/site/app.js index 2f206fd..4c8f379 100644 --- a/site/app.js +++ b/site/app.js @@ -10,17 +10,24 @@ const stories = { reference: "The known-good control satisfies every recorded check. Compare its behavior with a deliberately faulty implementation to audit the grader.", }; let audits = {}, pack = "support", evaluation, selectedCase; +const auditDigests = {}; let comparison, baseline, current; const pretty = (data) => JSON.stringify(data, null, 2); const percent = (value) => value === null ? "Unassessed" : `${Number((value * 100).toFixed(4))}%`; const repetitions = {}; -async function fetchRecord(path) { +async function fetchRecord(path, withIdentity = false) { const controller = new AbortController(); const timer = setTimeout(() => controller.abort(), 15000); try { const response = await fetch(path, {signal: controller.signal}); if (!response.ok) throw new Error(`Evidence request failed (${response.status})`); - return await response.json(); + if (!withIdentity) return await response.json(); + const bytes = await response.arrayBuffer(); + const record = JSON.parse(new TextDecoder("utf-8", {fatal: true}).decode(bytes)); + const digest = globalThis.crypto?.subtle + ? await crypto.subtle.digest("SHA-256", bytes) : null; + return {record, sha256: digest + ? Array.from(new Uint8Array(digest), byte => byte.toString(16).padStart(2, "0")).join("") : null}; } finally { clearTimeout(timer); } @@ -150,6 +157,10 @@ function choosePack(nextPack, view) { if (view?.control && [...select.options].some(option => option.value === view.control)) select.value = view.control; $("report-link").href = `${packs[pack].path}/index.html`; $("download-link").href = `${packs[pack].path}/audit.json`; + $("share-case").disabled = !auditDigests[pack]; + $("audit-identity").textContent = auditDigests[pack] + ? `Loaded audit SHA-256: ${auditDigests[pack].slice(0, 16)}… Full fingerprint in the reproduction record.` + : "Audit fingerprint unavailable. Open this page over HTTPS to share an evidence-bound link; the report and downloads still work."; chooseControl(view); } function chooseControl(view) { @@ -182,6 +193,7 @@ function chooseControl(view) { $("cases").append(button); }); $("provenance").textContent = pretty({ + audit_sha256: auditDigests[pack], task: evaluation.task, created_at: evaluation.created_at, seeds: evaluation.seeds, evalarc_version: evaluation.evalarc_version, candidate_sha256: evaluation.candidate_sha256, grader_sha256: evaluation.grader_sha256, cases_sha256: evaluation.cases_sha256, @@ -229,19 +241,26 @@ function showStep() { } function viewFragment() { return "#" + new URLSearchParams({ - v: "1", pack, control: $("control").value, case: selectedCase.case_id, + v: "2", audit: auditDigests[pack], pack, control: $("control").value, case: selectedCase.case_id, seed: String(selectedCase.seed), step: $("trace-panel").hidden ? "0" : $("trace-step").value, }); } function rememberView() { $("share-url").hidden = true; $("share-status").textContent = ""; - try { history.replaceState(null, "", viewFragment()); } catch { /* Restricted embeds can still copy links. */ } + if (auditDigests[pack]) { + try { history.replaceState(null, "", viewFragment()); } catch { /* Restricted embeds can still copy links. */ } + } } function restoreView(focus = true) { if (!location.hash.includes("=")) return false; const params = new URLSearchParams(location.hash.slice(1)); - if (location.hash.length > 2048 || params.get("v") !== "1" || !Object.hasOwn(packs, params.get("pack"))) { + const version = params.get("v"); + const keys = new Set(["v", "audit", "pack", "control", "case", "seed", "step"]); + if (location.hash.length > 2048 || !["1", "2"].includes(version) || !Object.hasOwn(packs, params.get("pack")) || + [...params.keys()].some(key => !keys.has(key) || params.getAll(key).length !== 1) || + (version === "2" && !/^[a-f0-9]{64}$/.test(params.get("audit") || "")) || + (version === "1" && params.has("audit"))) { $("share-status").textContent = "This evidence link is not supported. Choose a task, control and case below."; return false; } @@ -250,6 +269,13 @@ function restoreView(focus = true) { $("share-status").textContent = `The linked ${requestedPack} evidence is unavailable. Retry the missing task pack to open this view.`; return false; } + if (version === "2" && params.get("audit") !== auditDigests[requestedPack]) { + $("share-url").hidden = true; + $("share-status").textContent = auditDigests[requestedPack] + ? `Evidence changed: this link expects audit ${params.get("audit").slice(0, 16)}…, but the loaded audit is ${auditDigests[requestedPack].slice(0, 16)}…. The linked view was not restored. Choose a case to review the current recording or open the original saved audit.` + : "This link's audit identity could not be checked. The linked view was not restored. Open this page over HTTPS or use the original saved audit."; + return false; + } const step = Number(params.get("step")); const view = {control: params.get("control"), case: params.get("case"), seed: params.get("seed"), step: Number.isSafeInteger(step) && step >= 0 ? step : 0}; @@ -259,7 +285,9 @@ function restoreView(focus = true) { (selectedCase.trace?.length ? Number($("trace-step").value) === view.step : view.step === 0); $("share-url").hidden = true; $("share-status").textContent = exact - ? "Shared evidence restored. The link selects a recorded case and trace step." + ? version === "2" + ? "Shared evidence restored. The loaded audit bytes match the link's SHA-256; this checks content identity, not authorship." + : "Legacy link: showing the selected case in the current recording. The original audit identity was not recorded in this link." : "Some link values were outside this recording. Showing the nearest available view; check the selected case and step."; if (focus) $("case-title").focus(); return true; @@ -270,7 +298,7 @@ $("share-case").addEventListener("click", async () => { try { await navigator.clipboard.writeText(url.href); $("share-url").hidden = true; - $("share-status").textContent = "Evidence link copied: task, control, seed, case and trace step."; + $("share-status").textContent = "Evidence link copied with the loaded audit's SHA-256, task, control, seed, case and trace step."; } catch { $("share-url").hidden = false; $("share-url").value = url.href; @@ -327,10 +355,11 @@ for (const name of Object.keys(packs)) $(name + "-task").addEventListener("click }); async function loadAudits() { await Promise.allSettled(Object.entries(packs).filter(([name]) => !audits[name]).map(async ([name, config]) => { - const record = await fetchRecord(`${config.path}/audit.json`); + const {record, sha256} = await fetchRecord(`${config.path}/audit.json`, true); if (!Array.isArray(record.reference?.cases) || !Array.isArray(record.mutants) || record.mutants.some(row => !Array.isArray(row.evaluation?.cases))) throw new Error("Incomplete audit evidence"); audits[name] = record; + auditDigests[name] = sha256; $(name + "-task").disabled = false; })); const available = Object.keys(audits), missing = Object.keys(packs).filter(name => !audits[name]); diff --git a/site/index.html b/site/index.html index 0e212a0..0055a69 100644 --- a/site/index.html +++ b/site/index.html @@ -147,6 +147,7 @@

A perfect score.
How much coverage remains?

+

diff --git a/src/evalarc/__init__.py b/src/evalarc/__init__.py index 19e3521..cc921c4 100644 --- a/src/evalarc/__init__.py +++ b/src/evalarc/__init__.py @@ -1,3 +1,3 @@ """Auditable evaluations for AI agents.""" -__version__ = "0.10.1" +__version__ = "0.10.2"