diff --git a/apps/presentation/dashboard/src/features/personal-workspace/goal-research-analysis.tsx b/apps/presentation/dashboard/src/features/personal-workspace/goal-research-analysis.tsx index dc039a9725..6548c9447f 100644 --- a/apps/presentation/dashboard/src/features/personal-workspace/goal-research-analysis.tsx +++ b/apps/presentation/dashboard/src/features/personal-workspace/goal-research-analysis.tsx @@ -150,7 +150,7 @@ export function useResearchAnalysis(goalId: string, active: boolean, zh: boolean }; const closeCitation = () => {citeGeneration.current++; setCitation(null); setCitationError(""); setCitationBusy(false); requestAnimationFrame(() => sourceTrigger.current?.focus());}; return {readOnly: Boolean(readSourceUrl), citation, citationBusy, citationError, sourcePanel, readCitation, closeCitation, goalId, zh, location, snapshot, pack, readError, error: error || readError, loading, pending, cancelPending, recordedRunning, panel, trigger, read, show, execute, cancel, - close: () => {citeGeneration.current++; setCitation(null); setCitationError(""); setCitationBusy(false); navigate(false); requestAnimationFrame(() => trigger.current?.focus());}, select: (id: string) => navigate(location.open, id)}; + close: () => {citeGeneration.current++; setCitation(null); setCitationError(""); setCitationBusy(false); navigate(false); requestAnimationFrame(() => trigger.current?.focus());}, select: (id: string) => navigate(location.open, id, true)}; } type Controller = ReturnType; diff --git a/apps/presentation/dashboard/src/features/personal-workspace/goal-research-progress.css b/apps/presentation/dashboard/src/features/personal-workspace/goal-research-progress.css index 6ac1de29bf..d68df2a2ca 100644 --- a/apps/presentation/dashboard/src/features/personal-workspace/goal-research-progress.css +++ b/apps/presentation/dashboard/src/features/personal-workspace/goal-research-progress.css @@ -8,6 +8,20 @@ .research-progress select { min-width: 0; max-width: 400px; min-height: 44px; border: 1px solid var(--pw-line); border-radius: 6px; padding: 8px; background: var(--pw-bg); color: var(--pw-text); font: inherit; } .research-progress p, .research-progress small { color: var(--pw-muted); font-size: var(--text-caption); line-height: var(--leading-prose); } .research-progress summary { min-height: 44px; padding-block: 12px; cursor: pointer; font-size: var(--text-caption); } +.research-retained-improvement { margin-block-start: 20px; } +.research-retained-improvement h4 { margin: 0; font-size: var(--text-body); font-weight: 500; } +.research-retained-improvement summary { display: flex; align-items: baseline; flex-wrap: wrap; gap: 8px 12px; list-style: none; } +.research-retained-improvement summary::-webkit-details-marker { display: none; } +.research-retained-improvement summary strong { font-size: var(--text-title); font-weight: 600; font-variant-numeric: tabular-nums; } +.research-retained-improvement summary span { color: var(--pw-muted); font-size: var(--text-body); } +.research-retained-improvement details[open] summary svg { transform: rotate(180deg); } +.research-retained-improvement dl { margin: 0; font-size: var(--text-body); } +.research-retained-improvement dl > div { padding-block: 6px; } +.research-retained-improvement dt { color: var(--pw-muted); font-size: var(--text-caption); } +.research-retained-improvement dd { margin: 0; } +.research-retained-improvement dd button { display: inline-flex; align-items: center; gap: 8px; min-height: 44px; padding: 0; background: none; border: 0; color: var(--pw-text); font: inherit; cursor: pointer; } +.research-retained-improvement dd button:hover { text-decoration: underline; } +.research-retained-improvement summary:focus-visible, .research-retained-improvement dd button:focus-visible { outline: 2px solid var(--pw-blue); outline-offset: 2px; } .research-effect-metrics { display: grid; grid-template-columns: repeat(6, minmax(0, 1fr)); gap: 8px 24px; margin-block-start: 12px; } .research-effect-metric { min-width: 0; } .research-effect-metric[open] { grid-column: 1 / -1; } diff --git a/apps/presentation/dashboard/src/features/personal-workspace/goal-research-progress.tsx b/apps/presentation/dashboard/src/features/personal-workspace/goal-research-progress.tsx index d44d628297..8e5206dc77 100644 --- a/apps/presentation/dashboard/src/features/personal-workspace/goal-research-progress.tsx +++ b/apps/presentation/dashboard/src/features/personal-workspace/goal-research-progress.tsx @@ -30,6 +30,28 @@ export function verdictLabel(metric: TrajectoryMetric, verdict: AssessmentVerdic // Format the native owner's existing projection, without deriving a gain, // deciding adoption, or borrowing values from another pack. +function RetainedImprovement({native, zh, onCitation}: {native: NonNullable; zh: boolean; onCitation: (citation: AssessmentCitation, origin?: HTMLButtonElement) => void}) { + const outcomes = native.projection, improvement = outcomes.retained_improvement; + if (!improvement) return null; + const format = (value: number) => new Intl.NumberFormat(zh ? "zh-CN" : "en", {maximumSignificantDigits: 10, signDisplay: "exceptZero"}).format(value); + const label = zh ? "已采用成果增量" : "Retained improvement"; + return
+

{label}

+

{zh ? "固定基线与实际保留结果的原生指标差值，正数表示改善。越低越好的指标反向计算；回退到基线为零，缺少基线或保留证据为未知。" : "Native change from a fixed baseline to the actually retained result. Positive means improvement; lower-is-better metrics reverse the subtraction. Returning to baseline is zero; missing baseline or retention evidence is unknown."}

+

{zh ? "只代表所选比较，不是 LoopX 的独立贡献或原目标已达成；不能跨题、跨单位或重叠窗口相加。展开数值可看原文。" : "This comparison alone does not establish LoopX's causal contribution or objective completion. Do not sum across tasks, units or overlapping windows. Expand the value for original evidence."}

+
+ {native.applicability === "historical" ?

{zh ? "历史捕获 · 当前结果未核验" : "Historical capture · current result unverified"}

: null} +
+ {improvement.value === null ? "—" : format(improvement.value)} + {improvement.value === null ? (zh ? "未知" : "Unknown") : outcomes.metric.unit} · {outcomes.metric.name} +
+ {([[improvement.baseline, zh ? "比较基线" : "Comparison baseline"], [outcomes.retained, zh ? "实际保留" : "Actually retained"]] as const).map(([point, title]) =>
{title}
{point ? : (zh ? "未知" : "Unknown")}
)} +
{zh ? "保留依据" : "Retention evidence"}
{improvement.retention_citation ? : (zh ? "未采集" : "Uncaptured")}
+
{zh ? "最后候选差值" : "Final candidate change"}
{improvement.final_candidate_delta === null ? (zh ? "未知" : "Unknown") : `${format(improvement.final_candidate_delta)} ${outcomes.metric.unit}`} · {zh ? "不代表已采用收益" : "not realized retention"}
+
+
; +} + function NativeResults({native, zh, onCitation}: {native: ResearchProgressPack["native_outcomes"]; zh: boolean; onCitation: (citation: AssessmentCitation, origin?: HTMLButtonElement) => void}) { const outcomes = native?.projection; if (!native || !outcomes) return

{zh ? "原生测量 · 未采集" : "Native measurements · not captured"}

; @@ -54,7 +76,7 @@ function NativeResults({native, zh, onCitation}: {native: ResearchProgressPack["

{native.artifact}
{native.artifact_sha256}

{points.map(([key, cn, en]) => { const point = outcomes[key]; - return point ?
{zh ? cn : en}
{point.candidate_ref}
{zh ? "提交" : "Submitted"}: {point.submitted_at}
{zh ? "收到结果" : "Result received"}: {point.observed_at}
: null; + return point ?
{zh ? cn : en}
{point.candidate_ref}
{zh ? "提交" : "Submitted"}: {point.submitted_at ?? (zh ? "未知" : "Unknown")}
{zh ? "收到结果" : "Result received"}: {point.observed_at}
: null; })}
@@ -81,6 +103,7 @@ export function ResearchProgress({pack: p, packs, selectedId, error, zh, onSelec : (zh ? "所选来源暂不可读，未显示其他范围的结果。" : "The selected source could not be read. No result from another scope is shown.")} {error}

: null} {p ? <> {p.applicability === "historical" ?

{zh ? "历史分析：来源或方法已变化，不能当作当前判断。" : "Historical analysis: the source or method changed; these are not current judgments."}

: null} + {p.native_outcomes ? : null} {result ?
{TRAJECTORY_METRICS.map(metric => { @@ -114,13 +137,13 @@ export function ResearchProgress({pack: p, packs, selectedId, error, zh, onSelec })}
:

{zh ? "所选范围尚未分析" : "No analysis for the selected scope"}

} - {p.native_outcomes ? : null}
{zh ? "来源与用量" : "Sources and usage"}{p.native_outcomes ? "" : (zh ? " · 原生测量未采集" : " · Native measurements uncaptured")}{zh ? " · 完整成本未知" : " · Total cost unknown"}
{zh ? "题目 / 运行" : "Case / run"}
{p.scope.case_id}
{p.scope.run_id}
{zh ? "证据窗口" : "Evidence window"}
{p.scope.window}
{zh ? "取样规则" : "Selection rule"}
{p.selection.rule}
{zh ? "总体研究片段数" : "Population of episodes"}
{p.selection.population_episodes == null ? (zh ? "未知" : "Unknown") : number(p.selection.population_episodes)}
{result ? <>
{zh ? "分析方法" : "Analysis method"}
{result.method_version} · {zh ? "Agent 判读未校准" : "Uncalibrated Agent judgment"}
{zh ? "结果标识" : "Result key"}
{result.analysis_key}
{zh ? "分析限制" : "Limitations"}
{result.limitations.map((line, i) =>

{line}

)}
: null}
{zh ? "来源版本" : "Source revision"}
{p.scope.revision}
+ {p.native_outcomes ? : null}

{zh ? "用量与成本" : "Usage and cost"}{measured?.applicability === "historical" ? (zh ? " · 历史捕获" : " · Historical capture") : ""}

{zh ? "只读捕获，窗口可能与以上判读片段及下方 Explore 历史不同。" : "Read-only capture; its window can differ from the judgments above and Explore history below."}

diff --git a/docs/reference/trajectory-assessment.md b/docs/reference/trajectory-assessment.md index 1389b6565d..5756fb795d 100644 --- a/docs/reference/trajectory-assessment.md +++ b/docs/reference/trajectory-assessment.md @@ -233,6 +233,10 @@ contiguous submission indices starting at one. An invalid record uses `valid:false,value:null`; zero is a valid measured value, never a missing value. `retained_result_id:null` means retention is unknown, not that the mathematically best value was adopted. +An archived measurement may explicitly use `submitted_at:null` when only its +readback time is known; `observed_at` must still be inside the capture. This does +not invent an original submission time, a full evaluation duration or prefix +coverage. Known submission times retain all existing calendar/window checks. The existing typed benchmark-toolkit prepare owner returns a separate bounded `native_outcomes` projection: first valid submission-one value when the prefix @@ -258,8 +262,53 @@ Synthetic packaged native readback (not benchmark findings): ![Native outcome source disclosure](../development/assets/research-analysis/native-outcomes-desktop.png) [Narrow-screen source disclosure](../development/assets/research-analysis/native-outcomes-mobile.png). -The existing six-judgment first viewport is unchanged; these facts live in its -existing source disclosure, with costs in their separate disclosure. +Detailed native facts live in the existing source disclosure, with costs in +their separate disclosure. + +#### Quantified retained improvement + +To expose observed outcome value alongside the six process judgments, the same +provider may optionally add `comparison` to its native capture: + +```json +"comparison": { + "baseline_result_id": "result-1", + "retention_citation": {"source_id":"adoption-source", "first_line":4, "last_line":8} +} +``` + +The provider fixes a comparable baseline under the same native objective, +metric, unit and evaluation conditions; it must not select a baseline after +seeing the gain. Its result must be valid and present in this same scoped +capture, no later in submission order than the retained result. An incomplete +prefix may still contain an explicitly known baseline; that does not establish +the study's initial value. The retention citation must bind original evidence +of actual retained source/result identity, not a proposed decision or the +mathematically best score. Source binding checks provenance, not the truth or +comparability of provider claims. + +The TypeScript owner adds `retained_improvement`: baseline, retention citation, +signed `value` and separate `final_candidate_delta`. Higher-is-better uses +retained minus baseline; lower-is-better reverses the subtraction. For example, +native error 7 → retained 3 yields **+4 points**. Retaining 7 after rejecting a +candidate with error 9 yields **0**, with candidate delta **−2**; avoiding that +regression is not a new realized benefit. An actually retained regression is +negative. Baseline or retention may explicitly be `null`; missing baseline, +retained result or retention evidence leaves realized value `null`, never zero. +Invalid references, unbound citations and non-finite differences are rejected. +Omitting `comparison` preserves the previous projection shape and does not +invent a baseline from the first/best result. Native-only registration still +does not execute an analysis or alter its key. + +The Goal shows one compact **Retained improvement / 已采用成果增量** value before +the process judgments. Hover, keyboard focus or tap explains the metric; +expansion exposes both measured endpoints, retention evidence and candidate +change. Historical/unreadable provenance remains visible. The value describes +the declared native comparison, not objective completion, a general utility +score, matched LoopX uplift or causal attribution. Do not add across tasks, +units, overlapping windows or inherited results. Displayed precision is rounded; +original endpoint values remain readable in their citations. Pack selection +creates browser history so Back/Forward restores its scope and evidence. ### Solver capture registration diff --git a/examples/personal-workspace-browser/assets/retained-value-before.png b/examples/personal-workspace-browser/assets/retained-value-before.png new file mode 100644 index 0000000000..b4322a7099 Binary files /dev/null and b/examples/personal-workspace-browser/assets/retained-value-before.png differ diff --git a/examples/personal-workspace-browser/assets/retained-value-desktop.png b/examples/personal-workspace-browser/assets/retained-value-desktop.png new file mode 100644 index 0000000000..435202c78b Binary files /dev/null and b/examples/personal-workspace-browser/assets/retained-value-desktop.png differ diff --git a/examples/personal-workspace-browser/assets/retained-value-mobile.png b/examples/personal-workspace-browser/assets/retained-value-mobile.png new file mode 100644 index 0000000000..8c448f4eae Binary files /dev/null and b/examples/personal-workspace-browser/assets/retained-value-mobile.png differ diff --git a/examples/personal-workspace-browser/research-history.mjs b/examples/personal-workspace-browser/research-history.mjs index a13b56fa1f..19657cb5a8 100644 --- a/examples/personal-workspace-browser/research-history.mjs +++ b/examples/personal-workspace-browser/research-history.mjs @@ -52,6 +52,15 @@ function analysisReadback(selected = "a") { snapshot.pack.native_outcomes.projection.final_valid = {...snapshot.pack.native_outcomes.projection.final_valid, submission_index: 4, result_id: "native-later", candidate_ref: "candidate-later", value: 5}; } + if (["a", "b", "d"].includes(selected)) { + const native = snapshot.pack.native_outcomes.projection; + native.retained_improvement = { + baseline: selected === "d" ? null : selected === "b" ? native.retained : native.initial, + retention_citation: {source_id: "source", first_line: 1, last_line: 2}, + value: selected === "d" ? null : selected === "b" ? 0 : 4, + final_candidate_delta: selected === "d" ? null : selected === "b" ? -2 : 4, + }; + } if (selected === "d") {snapshot.pack.result = null; snapshot.pack.job = null; snapshot.pack.native_outcomes.projection.initial = null;} return snapshot; } @@ -448,8 +457,20 @@ Path(sys.argv[1]).write_text(json.dumps({'goal_id':goal,'display_name':'Syntheti await progress.locator(`.research-progress-scope[data-pack-id="${"a".repeat(64)}"]`).waitFor(); const basis = progress.locator('.research-progress-basis'); assert.equal(await basis.evaluate(e => e.open), false); + const improvement = progress.getByRole('region', {name: '已采用成果增量', exact: true}); + assert.equal(await improvement.locator('[data-retained-improvement]').textContent(), '+4'); + await improvement.getByRole('button', {name: '成果增量说明', exact: true}).focus(); + await improvement.getByRole('tooltip').waitFor({state: 'visible'}); + await page.keyboard.press('Escape'); + await improvement.locator('summary').click(); + await improvement.getByRole('button', {name: '查看原文', exact: true}).click(); + await citedOriginal.waitFor(); + await citedOriginal.press('Escape'); + await history.getByRole('button', {name: '关闭分析', exact: true}).click(); + await improvement.locator('summary').click(); + await basis.locator(':scope > summary').click(); const nativeResults = progress.getByRole('region', {name: '原生测量', exact: true}); - assert.match(await nativeResults.innerText(), /首次提交.*7.*提交 1/s, 'Native measurements need no disclosure before reading'); + assert.match(await nativeResults.innerText(), /首次提交.*7.*提交 1/s, 'Detailed native measurements use the existing source disclosure'); assert.match(await nativeResults.innerText(), /原生保留结果.*3.*提交 3/s); assert.match(await nativeResults.innerText(), /越低越好/); await nativeResults.getByRole('button', {name: /首次提交.*查看原始证据/}).scrollIntoViewIfNeeded(); @@ -462,6 +483,7 @@ Path(sys.argv[1]).write_text(json.dumps({'goal_id':goal,'display_name':'Syntheti assert.match(await citedOriginal.innerText(), /Synthetic original evidence/); await citedOriginal.press('Escape'); await history.getByRole('button', {name: '关闭分析', exact: true}).click(); + await basis.locator(':scope > summary').click(); await benefitMetric.locator('summary').click(); assert.match(await progress.innerText(), /0 \/ 1/); assert.match(await progress.innerText(), /1 \/ 2/); @@ -493,6 +515,8 @@ Path(sys.argv[1]).write_text(json.dumps({'goal_id':goal,'display_name':'Syntheti await page.evaluate(() => new Promise(resolve => requestAnimationFrame(() => requestAnimationFrame(resolve)))); assert.equal(await analysis.locator("header p").first().textContent(), "Synthetic analysis B", "A delayed same-Goal read cannot undo a pack selection"); assert.match(await progress.innerText(), /历史捕获|historical/i); + assert.equal(await improvement.locator('[data-retained-improvement]').textContent(), '0', 'A rejected candidate does not create realized gain'); + await basis.locator(':scope > summary').click(); assert.match(await nativeResults.innerText(), /历史原生测量.*当前结果未核验/s, 'Stale measurements never appear current'); assert.match(await nativeResults.locator('[data-native-point="retained"]').innerText(), /3.*提交 3/s); assert.match(await nativeResults.locator('[data-native-point="final_valid"]').innerText(), /5.*提交 4/s, 'The last submission is not silently presented as retained or best'); @@ -503,14 +527,8 @@ Path(sys.argv[1]).write_text(json.dumps({'goal_id':goal,'display_name':'Syntheti await page.screenshot({path: resolve(outputDir, "research-analysis-mobile.png"), animations: "disabled"}); // Browser history changes source identity too. While A is delayed or // unavailable, B's interpretation/cost must not be shown under A's URL. - await page.evaluate(({goal, a, b}) => { - const url = new URL(location.href); - url.searchParams.set("research_analysis_goal", goal); - url.searchParams.set("research_pack", a); - history.replaceState(history.state, "", url); - url.searchParams.set("research_pack", b); - history.pushState(history.state, "", url); - }, {goal, a: "a".repeat(64), b: "b".repeat(64)}); + // Use the real selector-created history entry, rather than fabricating + // pushState here: source selection itself must preserve Back/Forward. const backStarted = new Promise(resolve => {slowStarted = resolve;}); const backDelivered = new Promise(resolve => {oldDelivered = resolve;}); holdOldAnalysis = unavailableAnalysis = true; @@ -553,6 +571,7 @@ Path(sys.argv[1]).write_text(json.dumps({'goal_id':goal,'display_name':'Syntheti 'An all-unknown but eligible metric has no declared exclusion'); assert.doesNotMatch(await progress.innerText(), /0%/); assert.equal(await nativeResults.count(), 0, 'A missing native capture cannot borrow the previous scope'); + assert.equal(await improvement.count(), 0, 'An absent comparison preserves the prior process-only surface'); assert.match(await progress.innerText(), /原生测量.*未采集/); await history.getByRole("button", {name: "查看分析", exact: true}).click(); await analysis.locator("header p").first().filter({hasText: /^Synthetic analysis C$/}).waitFor(); @@ -562,7 +581,9 @@ Path(sys.argv[1]).write_text(json.dumps({'goal_id':goal,'display_name':'Syntheti assert.match(await progress.innerText(), /所选范围尚未分析/); assert.equal(await progress.locator('.research-effect-metrics').count(), 0, 'An unanalysed cost pack never borrows another pack’s judgments'); assert.equal(await analysis.count(), 0, 'Choosing an unanalysed source never requests analysis'); - assert.match(await nativeResults.innerText(), /原生保留结果.*3.*提交 3/s, 'Native facts do not require an analyzer result or disclosure'); + assert.equal(await improvement.locator('[data-retained-improvement]').textContent(), '—', 'Missing comparison evidence is unknown, never zero'); + await basis.locator(':scope > summary').click(); + assert.match(await nativeResults.innerText(), /原生保留结果.*3.*提交 3/s, 'Native facts do not require an analyzer result'); assert.match(await nativeResults.innerText(), /部分窗口，起点未知/); assert.match(await nativeResults.locator('[data-native-point="initial"]').innerText(), /— 未知/); const nativeCitation = nativeResults.getByRole('button', {name: /原生保留结果.*查看原始证据/}); diff --git a/loopx/capabilities/benchmark_toolkit/trajectory_assessment_store.py b/loopx/capabilities/benchmark_toolkit/trajectory_assessment_store.py index 5f56324a62..5a44ac6eda 100644 --- a/loopx/capabilities/benchmark_toolkit/trajectory_assessment_store.py +++ b/loopx/capabilities/benchmark_toolkit/trajectory_assessment_store.py @@ -464,6 +464,11 @@ def read_citation( citations = [c for a in (result or {}).get("assessments", []) for c in a["citations"]] native = (view.get("native_outcomes") or {}).get("projection") or {} native_citations = [native[key]["citation"] for key in ("initial", "best_valid", "retained", "final_valid") if native.get(key)] + improvement = native.get("retained_improvement") or {} + if improvement.get("baseline"): + native_citations.append(improvement["baseline"]["citation"]) + if improvement.get("retention_citation"): + native_citations.append(improvement["retention_citation"]) requested = {"source_id": source_id, "first_line": first_line, "last_line": last_line} if not any( c["source_id"] == source_id diff --git a/loopx/control_plane/capabilities/trajectory_native_outcomes.ts b/loopx/control_plane/capabilities/trajectory_native_outcomes.ts index cac3e837d2..7f344cb0ab 100644 --- a/loopx/control_plane/capabilities/trajectory_native_outcomes.ts +++ b/loopx/control_plane/capabilities/trajectory_native_outcomes.ts @@ -4,7 +4,7 @@ import type {AssessmentCitation, TrajectoryInput} from "./trajectory_assessment_ export interface NativeOutcomePoint { submission_index: number; result_id: string; candidate_ref: string; - submitted_at: string; observed_at: string; value: number; citation: AssessmentCitation; + submitted_at: string | null; observed_at: string; value: number; citation: AssessmentCitation; } export interface TrajectoryNativeOutcomes { method: "source_bound_native_outcomes_v1"; @@ -13,6 +13,10 @@ export interface TrajectoryNativeOutcomes { sample_count: number; valid_count: number; initial: NativeOutcomePoint | null; best_valid: NativeOutcomePoint | null; retained: NativeOutcomePoint | null; final_valid: NativeOutcomePoint | null; + retained_improvement?: { + baseline: NativeOutcomePoint | null; retention_citation: AssessmentCitation | null; + value: number | null; final_candidate_delta: number | null; + }; limitations: string[]; } @@ -43,6 +47,14 @@ function timestamp(value: unknown, label: string): string { if (new Date(value).toISOString() !== canonical) fail(`${label} must be a UTC timestamp`); return value; } +function boundCitation(value: unknown, input: TrajectoryInput): AssessmentCitation { + const citation = object(value, "native citation"), source = input.sources.find(s => s.id === citation.source_id); + const first = citation.first_line, last = citation.last_line; + if (!source || !Number.isSafeInteger(first) || !Number.isSafeInteger(last) + || Number(first) < source.first_line || Number(last) < Number(first) + || Number(last) >= source.first_line + source.text.split("\n").length) fail("native citation is outside registered evidence"); + return {source_id: source.id, first_line: Number(first), last_line: Number(last)}; +} /** The provider declares validity/retention. This owner checks binding and ranks * valid measurements within that declared capture, never certifying a study. */ @@ -67,19 +79,16 @@ export function projectTrajectoryNativeOutcomes(value: unknown, input: Trajector const result = identifier(sample.result_id, "native result"), candidate = identifier(sample.candidate_ref, "native candidate"); if (seen.has(result)) fail("native result identity is duplicated"); seen.add(result); - const submitted = timestamp(sample.submitted_at, "native submitted_at"), observed = timestamp(sample.observed_at, "native observed_at"); - if (Date.parse(submitted) < Date.parse(start) || Date.parse(observed) > Date.parse(end) - || Date.parse(submitted) > Date.parse(observed)) fail("native result lies outside its capture or precedes submission"); + const submitted = sample.submitted_at === null ? null : timestamp(sample.submitted_at, "native submitted_at"); + const observed = timestamp(sample.observed_at, "native observed_at"); + if (Date.parse(observed) < Date.parse(start) || Date.parse(observed) > Date.parse(end) + || submitted && (Date.parse(submitted) < Date.parse(start) || Date.parse(submitted) > Date.parse(observed))) fail("native result lies outside its capture or precedes submission"); if (typeof sample.valid !== "boolean" || sample.valid && (typeof sample.value !== "number" || !Number.isFinite(sample.value)) || !sample.valid && sample.value !== null) fail("native validity and measured value must agree; missing/invalid is not zero"); - const citation = object(sample.citation, "native citation"), source = input.sources.find(s => s.id === citation.source_id); - const first = citation.first_line, last = citation.last_line; - if (!source || !Number.isSafeInteger(first) || !Number.isSafeInteger(last) - || Number(first) < source.first_line || Number(last) < Number(first) - || Number(last) >= source.first_line + source.text.split("\n").length) fail("native citation is outside registered evidence"); + const citation = boundCitation(sample.citation, input); if (sample.valid) valid.push({submission_index: Number(index), result_id: result, candidate_ref: candidate, submitted_at: submitted, observed_at: observed, value: sample.value as number, - citation: {source_id: source.id, first_line: Number(first), last_line: Number(last)}}); + citation}); } let best: NativeOutcomePoint | null = null; for (const point of valid) if (!best || (metric.higher_is_better ? point.value > best.value : point.value < best.value)) best = point; @@ -87,10 +96,28 @@ export function projectTrajectoryNativeOutcomes(value: unknown, input: Trajector if (retainedId !== null) identifier(retainedId, "retained native result"); const retained = valid.find(point => point.result_id === retainedId) ?? null; if (retainedId !== null && !retained) fail("retained native result must be a valid result in this capture"); + let improvement: TrajectoryNativeOutcomes["retained_improvement"]; + if (Object.hasOwn(raw, "comparison")) { + const comparison = object(raw.comparison, "native comparison"), baselineId = comparison.baseline_result_id; + if (baselineId !== null) identifier(baselineId, "baseline native result"); + const baseline = valid.find(point => point.result_id === baselineId) ?? null; + if (baselineId !== null && !baseline) fail("baseline native result must be a valid result in this capture"); + if (baseline && retained && baseline.submission_index > retained.submission_index) fail("baseline cannot follow the retained result"); + const retention = comparison.retention_citation === null ? null : boundCitation(comparison.retention_citation, input); + const delta = (point: NativeOutcomePoint | null) => { + if (!baseline || !point) return null; + const difference = metric.higher_is_better ? point.value - baseline.value : baseline.value - point.value; + if (!Number.isFinite(difference)) fail("native improvement must be finite"); + return difference === 0 ? 0 : difference; + }; + improvement = {baseline, retention_citation: retention, value: retention ? delta(retained) : null, + final_candidate_delta: delta(valid.at(-1) ?? null)}; + } return {method: "source_bound_native_outcomes_v1", scope: input.scope, start, end, complete: raw.complete, metric: {name, unit, higher_is_better: metric.higher_is_better}, sample_count: raw.samples.length, valid_count: valid.length, initial: raw.complete ? valid.find(point => point.submission_index === 1) ?? null : null, best_valid: best, retained, final_valid: valid.at(-1) ?? null, + ...(improvement ? {retained_improvement: improvement} : {}), limitations: ["Provider-recorded measurements in the declared capture; not study countability or a causal effect.", "Validity and retained selection belong to the native provider, not model judgments or Todo status.", "Initial requires a complete prefix and a valid first submission; invalid or missing results are not zero.", diff --git a/tests/capabilities/test_trajectory_assessment_store.py b/tests/capabilities/test_trajectory_assessment_store.py index 1421db0bd8..6f5507e5e1 100644 --- a/tests/capabilities/test_trajectory_assessment_store.py +++ b/tests/capabilities/test_trajectory_assessment_store.py @@ -189,6 +189,46 @@ def test_native_results_register_through_real_cli_and_read_without_analysis(pack assert not list((scope["runtime_root"] / "benchmark" / "trajectory-assessments").glob("*/jobs/*")) +def test_native_improvement_citations_are_readable_without_model_execution(pack): + scope, _, _, input_path, sources, _ = pack + evidence = "Native results\nolder best: 100\nbaseline: 12\nretained: 15\nApplied retained source; verified hash\n" + (sources / "result.txt").write_text(evidence) + data = json.loads(input_path.read_text()) + data["sources"][0].update(first_line=1, text=evidence, + artifact_sha256=hashlib.sha256(evidence.encode()).hexdigest()) + input_path.write_text(json.dumps(data)) + native = { + "schema_version": "trajectory_native_outcomes_v1", "scope": data["scope"], + "start": "2026-01-01T00:00:00Z", "end": "2026-01-01T00:10:00Z", "complete": False, + "metric": {"name": "utility", "unit": "points", "higher_is_better": True}, + "retained_result_id": "result-9", + "comparison": {"baseline_result_id": "result-7", + "retention_citation": {"source_id": "source", "first_line": 5, "last_line": 5}}, + "samples": [{"submission_index": index, "result_id": f"result-{index}", "candidate_ref": f"candidate-{index}", + "submitted_at": "2026-01-01T00:01:00Z", "observed_at": "2026-01-01T00:02:00Z", + "valid": True, "value": value, "citation": {"source_id": "source", "first_line": line, "last_line": line}} + for index, value, line in [(2, 100, 2), (7, 12, 3), (9, 15, 4)]], + } + path = sources / "comparison.json" + path.write_text(json.dumps(native)) + registered = register_pack(**scope, input_path=input_path, source_root=sources, + title="Source-bound retained improvement", native_outcomes_json=path.name) + operation = {**scope, "pack_id": registered["pack_id"], "analysis_key": registered["analysis_key"]} + view = inspect_pack(**scope, pack_id=registered["pack_id"]) + assert view["native_outcomes"]["projection"]["retained_improvement"]["value"] == 3 + assert view["native_outcomes"]["projection"]["initial"] is None + assert view["result"] is None and view["job"] is None + assert read_citation(**operation, source_id="source", first_line=3, last_line=3)["text"] == "baseline: 12" + assert read_citation(**operation, source_id="source", first_line=5, last_line=5)["text"] == "Applied retained source; verified hash" + with pytest.raises(PackUnavailable, match="citation_not_in_result"): + read_citation(**operation, source_id="source", first_line=1, last_line=1) + path.write_text("{}") + old = inspect_pack(**scope, pack_id=registered["pack_id"]) + assert old["native_outcomes"]["applicability"] == "historical" + assert old["native_outcomes"]["projection"]["retained_improvement"]["value"] == 3 + assert read_citation(**operation, source_id="source", first_line=5, last_line=5)["applicability"] == "historical" + + def test_native_capture_cannot_borrow_another_run_or_escape_source_root(pack): scope, _, _, input_path, sources, _ = pack path = sources / "wrong-native.json" diff --git a/tests/control_plane_ts/trajectory_native_outcomes.test.ts b/tests/control_plane_ts/trajectory_native_outcomes.test.ts index df176b9763..72f43b86a8 100644 --- a/tests/control_plane_ts/trajectory_native_outcomes.test.ts +++ b/tests/control_plane_ts/trajectory_native_outcomes.test.ts @@ -21,10 +21,62 @@ test("native facts preserve first quality, actual retention and submission order assert.equal(native.initial?.value,2); assert.equal(native.best_valid?.value,1); assert.equal(native.retained?.result_id,"result-3"); assert.equal(native.final_valid?.result_id,"result-3"); assert.equal(native.sample_count,3); assert.equal(native.valid_count,3); + assert.equal(Object.hasOwn(native,"retained_improvement"),false,"An absent comparison preserves the old response shape"); assert.equal(prepareTrajectoryAssessment({input,config,native_outcomes:capture()}).analysis_key, prepareTrajectoryAssessment({input,config}).analysis_key,"native readback does not rerun unchanged analysis"); assert.equal(Object.hasOwn(prepareTrajectoryAssessment({input,config}),"native_outcomes"),false); }); +test("a declared baseline measures adopted improvement in native units, including lower-is-better and losses",()=>{ + const raw={...capture(), comparison:{baseline_result_id:"result-1",retention_citation:{source_id:"s",first_line:3,last_line:3}}}; + let native=project(raw); + assert.equal(native.retained_improvement?.value,1); + assert.equal(native.retained_improvement?.baseline?.value,2); + raw.metric.higher_is_better=true; + assert.equal(project(raw).retained_improvement?.value,-1,"An adopted regression is negative, not clamped away"); + raw.complete=false;raw.samples=raw.samples.slice(1);raw.comparison.baseline_result_id="result-2"; + native=project(raw); + assert.equal(native.initial,null,"An explicit comparison does not invent a complete prefix"); + assert.equal(native.retained_improvement?.value,-4); +}); +test("rejecting a candidate and retaining the baseline realizes zero, not an avoided-loss benefit",()=>{ + const raw={...capture(), comparison:{baseline_result_id:"result-1",retention_citation:{source_id:"s",first_line:1,last_line:1}}}; + raw.metric.higher_is_better=true;raw.retained_result_id="result-1"; + assert.equal(project(raw).retained_improvement?.value,0); + assert.equal(project(raw).retained_improvement?.final_candidate_delta,-1); + assert.equal(project(raw).best_valid?.value,5,"Best observed candidate does not override actual retention"); +}); +test("unknown baselines and unobserved retention cannot produce realized improvement",()=>{ + const raw:any={...capture(), comparison:{baseline_result_id:null,retention_citation:{source_id:"s",first_line:3,last_line:3}}}; + assert.equal(project(raw).retained_improvement?.value,null,"No fallback to submission one"); + raw.comparison.baseline_result_id="result-1";raw.comparison.retention_citation=null; + assert.equal(project(raw).retained_improvement?.value,null); + raw.comparison.retention_citation={source_id:"s",first_line:3,last_line:3};raw.retained_result_id=null; + assert.equal(project(raw).retained_improvement?.value,null); + assert.equal(project(raw).retained_improvement?.final_candidate_delta,1,"Observed candidate change is separate from retention"); +}); +test("an archived result may have unknown submission time without inventing a clock or losing its measurement",()=>{ + const raw:any={...capture(), comparison:{baseline_result_id:"result-1",retention_citation:{source_id:"s",first_line:3,last_line:3}}}; + raw.samples[0].submitted_at=null; + assert.equal(project(raw).retained_improvement?.value,1); + assert.equal(project(raw).retained_improvement?.baseline?.submitted_at,null); + raw.samples[0].observed_at="2025-12-31T23:59:59Z"; + assert.throws(()=>project(raw),/outside its capture/); +}); +test("comparisons reject foreign, invalid or later baselines, unbound retention evidence and overflow",()=>{ + for (const mutate of [ + (x:any)=>x.comparison.baseline_result_id="foreign", + (x:any)=>{x.samples[0].valid=false;x.samples[0].value=null;}, + (x:any)=>{x.comparison.baseline_result_id="result-3";x.retained_result_id="result-1";}, + (x:any)=>x.comparison.retention_citation.source_id="foreign", + (x:any)=>x.comparison.retention_citation.last_line=4, + (x:any)=>delete x.comparison.baseline_result_id, + (x:any)=>delete x.comparison.retention_citation, + (x:any)=>{x.samples[0].value=-Number.MAX_VALUE;x.samples[2].value=Number.MAX_VALUE;}, + ]) { + const raw:any={...capture(), comparison:{baseline_result_id:"result-1",retention_citation:{source_id:"s",first_line:3,last_line:3}}}; + mutate(raw);assert.throws(()=>project(raw)); + } +}); test("higher-is-better is explicit; best measurement does not invent retained selection",()=>{ const raw=capture();raw.metric.higher_is_better=true;raw.retained_result_id="result-1"; raw.metric.name="Native throughput (%)"; raw.metric.unit="ops/s";