From 691d863c04204f3851cc5148cccd0a096654f72e Mon Sep 17 00:00:00 2001 From: Fake Copilot Date: Wed, 11 Nov 2026 03:03:44 -0700 Subject: [PATCH] Observe bounded native autopilot continuations Add an optional two-opportunity boundary to the existing native evaluator. Preserve the initiating send, require correlated final responses and turn ends, pass native continuations unchanged, and stop at the selected boundary without changing the default idle behavior. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: cf89240b-a65d-414a-9123-5c1dc9c40c7d --- .../scripts/native_workflow.py | 32 ++- .../scripts/native_workflow_probe.mjs | 255 +++++++++++++++--- .../scripts/native_workflow_probe.test.mjs | 212 +++++++++++++-- .../scripts/session_restore.py | 13 + skills/skill-evaluation/scripts/skill_eval.py | 38 ++- 5 files changed, 486 insertions(+), 64 deletions(-) diff --git a/skills/skill-evaluation/scripts/native_workflow.py b/skills/skill-evaluation/scripts/native_workflow.py index 7e5ea95..624031f 100644 --- a/skills/skill-evaluation/scripts/native_workflow.py +++ b/skills/skill-evaluation/scripts/native_workflow.py @@ -165,13 +165,17 @@ def owned(source): "schema_version", "purpose", "running_in_interactive_mode", "session_capabilities", "cli_preamble", "executable_path", "locale", "timeout_ms", "package_root", "workflow_commit", "workflow_sha256", "canary_receipt", "canary_checks", - } | ({"skill_char_budget"} if isinstance(value, dict) and "skill_char_budget" in value else set()), + } | ({key for key in ("skill_char_budget", "native_opportunity_limit") if key in value} + if isinstance(value, dict) else set()), "historical continuation") + if "native_opportunity_limit" in value and ( + type(value["native_opportunity_limit"]) is not int or value["native_opportunity_limit"] != 2): + raise ValueError("native_opportunity_limit must be the integer 2 when explicitly selected") if "skill_char_budget" in value and ( type(value["skill_char_budget"]) is not int or not 1 <= value["skill_char_budget"] <= 4294967295): raise ValueError("skill_char_budget must be an integer in 1..4294967295") if (type(value["schema_version"]) is not int or value["schema_version"] != 1 - or value["purpose"] not in ("setup-canary", "historical") + or value["purpose"] not in ("setup-canary", "historical", "adapted-public-resume") or value["running_in_interactive_mode"] is not True or value["session_capabilities"] != CLI_CONTINUATION_CAPABILITIES or value["cli_preamble"] != CLI_CONTINUATION_PREAMBLE @@ -220,15 +224,23 @@ def owned(source): } if "skill_char_budget" in value: binding["interactive"]["skill_char_budget"] = value["skill_char_budget"] + capped = False if "context_fixture_file" in config: fixture = read_owner_json(owned(config["context_fixture_file"])) if "runtime_treatment" in fixture: - from session_restore import NORMALIZED_ACTIVE_CANARY - if (fixture["runtime_treatment"] != NORMALIZED_ACTIVE_CANARY - or value["purpose"] != "setup-canary" + from session_restore import NORMALIZED_ACTIVE_CANARY, CAPPED_PUBLIC_RESUME + capped = fixture["runtime_treatment"] == CAPPED_PUBLIC_RESUME + if ((not capped and (fixture["runtime_treatment"] != NORMALIZED_ACTIVE_CANARY + or value["purpose"] != "setup-canary")) + or (capped and ( + (value["purpose"], fixture.get("provenance")) not in ( + ("setup-canary", "synthetic-continuation-canary"), + ("adapted-public-resume", "source-bound-adapted-public-resume")))) or "recorded_autopilot_request" not in fixture): - raise ValueError("Normalized active treatment is setup-canary only") - binding["runtime_treatment"] = NORMALIZED_ACTIVE_CANARY + raise ValueError( + "Adapted capped continuation requires matching synthetic setup-canary or source-bound adapted-public-resume provenance" + if capped else "Normalized active treatment is setup-canary only") + binding["runtime_treatment"] = fixture["runtime_treatment"] if "recorded_autopilot_request" in fixture: if "active_objective" not in fixture or "completed_objective" in fixture: raise ValueError("Recorded autopilot continuation requires its active objective") @@ -237,6 +249,12 @@ def owned(source): "agentMode": "autopilot", "objective_status": "active", "send_surface": "session.rpc.send", "wait": False, } + if value["purpose"] == "adapted-public-resume" and not capped: + raise ValueError("Adapted public-resume purpose requires the capped treatment and source-bound provenance") + if "native_opportunity_limit" in value: + if not capped: + raise ValueError("native_opportunity_limit requires capped source-bound autopilot continuation") + binding["native_opportunity_limit"] = value["native_opportunity_limit"] if "operational_file" in config: binding["operational"] = operational_input(Path(config["operational_file"]), config)["binding"] configuration_sha256 = hashlib.sha256(json.dumps( diff --git a/skills/skill-evaluation/scripts/native_workflow_probe.mjs b/skills/skill-evaluation/scripts/native_workflow_probe.mjs index 8e37571..aee73cf 100644 --- a/skills/skill-evaluation/scripts/native_workflow_probe.mjs +++ b/skills/skill-evaluation/scripts/native_workflow_probe.mjs @@ -173,6 +173,7 @@ function profileState(config) { } const normalizedActiveCanary = "public-command-normalized-active-v1"; +const cappedPublicResume = "capped-public-resume-v1"; function contextFixture(config) { const path = config.context_fixture_file; @@ -196,8 +197,10 @@ function contextFixture(config) { if (active) keys.push("active_objective"); if (autopilot) keys.push("recorded_autopilot_request"); if (normalized) keys.push("runtime_treatment"); + if (fixture.runtime_treatment === cappedPublicResume) keys.push("provenance"); if (!isDeepStrictEqual(Object.keys(fixture).sort(), keys.sort()) - || (normalized && (!active || fixture.runtime_treatment !== normalizedActiveCanary)) + || (normalized && (!active + || ![normalizedActiveCanary, cappedPublicResume].includes(fixture.runtime_treatment))) || active !== autopilot || (active && completed) || [active && fixture.active_objective, autopilot && fixture.recorded_autopilot_request] .filter(value => value !== false).some(value => !value @@ -240,12 +243,23 @@ function contextFixture(config) { const completedStore = completed ? parseDataJson(readFileSync(fixture.completed_objective.path, "utf8")) : null; const activeStore = active ? parseDataJson(readFileSync(fixture.active_objective.path, "utf8")) : null; - if (normalized && (!/^\/synthetic\/[0-9a-f-]{36}\/events\.jsonl$/.test(canonical.source) + if (fixture.runtime_treatment === normalizedActiveCanary + && (!/^\/synthetic\/[0-9a-f-]{36}\/events\.jsonl$/.test(canonical.source) || activeStore.current.continuationCount !== 0 || !isDeepStrictEqual(activeStore.current.creditLimit, { credits: 100, creditsUsed: 0, creditsUsedNanoAiu: "0" }))) { throw new Error("Normalized active treatment requires the capped synthetic canary"); } + if (fixture.runtime_treatment === cappedPublicResume + && (!["synthetic-continuation-canary", "source-bound-adapted-public-resume"].includes(fixture.provenance) + || typeof canonical.source !== "string" || !canonical.source.startsWith("/") + || !canonical.source.endsWith("/events.jsonl") + || (fixture.provenance === "synthetic-continuation-canary" + ? !/^\/synthetic\/[0-9a-f-]{36}\/events\.jsonl$/.test(canonical.source) + : canonical.source.startsWith("/synthetic/")) + || Object.hasOwn(activeStore.current, "creditLimit"))) { + throw new Error("Adapted capped public resume requires an initially uncapped active objective and matching provenance"); + } const recordedRequest = autopilot ? parseDataJson(readFileSync(fixture.recorded_autopilot_request.path, "utf8")) : null; const ownerRoot = dirname(config.context_fixture_file); @@ -314,7 +328,9 @@ function contextFixture(config) { active_objective_native_state: compiled[3], autopilot_send: compiled[4], limitations: [...fixture.limitations, "Single recorded native event replay only; automatic producer leases, repeated continuations and timers are not reproduced", - "Native runtime regenerates transport current_datetime; the recorded datetime is not sent", + fixture.runtime_treatment === cappedPublicResume + ? "Opted-in candidate replay restores the exact source transformed prompt through the public SDK hook; zero-model setup sends no prompt" + : "Native runtime regenerates transport current_datetime; the recorded datetime is not sent", "Pinned single-send forwarding omits billable; no billing-fidelity claim, actual accounting unchanged", ] } : {}) }; } @@ -460,7 +476,8 @@ function operationalEntry(config) { { encoding: "utf8", env: { PATH: process.env.PATH ?? "" } })); } -export function continuationCorrespondence(before, after, events, messageId, raw, autopilotSend) { +export function continuationCorrespondence(before, after, events, messageId, raw, autopilotSend, exactTransformed, + nativePrompts) { if (after.length < before.length || before.some((event, index) => !isDeepStrictEqual(event, after[index]))) { throw new Error("Historical continuation mutated its native journal prefix"); } @@ -474,13 +491,23 @@ export function continuationCorrespondence(before, after, events, messageId, raw source: "autopilot", agentMode: "autopilot" }) && data?.content === "" && data.source === "autopilot" && data.agentMode === "autopilot" && wrapper && Number.isFinite(Date.parse(wrapper[1])) - && data.transformedContent.slice(wrapper[0].length) === raw; - if (users.length !== 1 || (autopilotSend ? !autopilotMatches + && data.transformedContent.slice(wrapper[0].length) === raw + && (exactTransformed === undefined || data.transformedContent === exactTransformed); + if ((nativePrompts === undefined ? users.length !== 1 : users.length !== nativePrompts.length + 1) + || (autopilotSend ? !autopilotMatches : (users[0].data?.content !== raw || users[0].data?.isAutopilotContinuation || users[0].data?.source)) || users[0].data?.messageId !== messageId || users[0].data?.attachments?.length || typeof users[0].data?.interactionId !== "string" || !users[0].data.interactionId) { throw new Error("Historical continuation requires exactly one unsubstituted root raw user event"); } + if (nativePrompts !== undefined && (nativePrompts.length > 1 || users.slice(1).some((event, index) => { + const later = event.data; + return !autopilotSend || later?.content !== "" || later.source !== "autopilot" || later.agentMode !== "autopilot" + || later.attachments?.length || typeof later.messageId !== "string" || !later.messageId + || later.messageId === messageId || typeof later.interactionId !== "string" || !later.interactionId + || typeof later.transformedContent !== "string" + || hash(later.transformedContent) !== nativePrompts[index].transformed_sha256; + }))) throw new Error("Unbound or substituted native continuation root event"); const primary = events.filter(event => root(event) && event.type === "system.message" && event.data?.interactionId === users[0].data.interactionId); if (!primary.length || primary.some(event => typeof event.data?.content !== "string" @@ -490,11 +517,15 @@ export function continuationCorrespondence(before, after, events, messageId, raw throw new Error("Fresh correlated primary system prompt did not verify interactive CLI identity"); } return { prefix_preserved: true, exactly_once: true, interactive_identity: true, + ...(nativePrompts === undefined ? {} : { native_continuation_passthrough: true }), ...(autopilotSend ? { recorded_autopilot_request: true } : {}), + ...(exactTransformed === undefined ? {} : { + recorded_transformed_content: true, recorded_transformed_sha256: hash(data.transformedContent), + }), primary_system_messages: primary, interaction_id: users[0].data.interactionId }; } -export function continuationCompletion(events, messageId) { +export function continuationCompletion(events, messageId, opportunity = false, originatingMessageId = messageId) { let interactionId; let assistant; let task; @@ -509,7 +540,7 @@ export function continuationCompletion(events, messageId) { && data.interactionId ? data.interactionId : undefined; assistant = task = undefined; } else if (event.type === "assistant.message") { - assistant = interactionId && typeof event.id === "string" && event.id && data?.originatingMessageId === messageId + assistant = interactionId && typeof event.id === "string" && event.id && data?.originatingMessageId === originatingMessageId && data.interactionId === interactionId ? event : undefined; const requests = Array.isArray(assistant?.data.toolRequests) ? assistant.data.toolRequests.filter(request => @@ -528,10 +559,26 @@ export function continuationCompletion(events, messageId) { task.completion = event; } else if (task?.completion && event.type === "assistant.turn_end") { task.turn_end = typeof event.id === "string" && event.id && data?.turnId === assistant.data.turnId ? event : undefined; + if (opportunity && task.turn_end && !task.invalid) { + return { kind: "task_complete", message: assistant, + task_complete: { request: task.request, start: task.start, + completion: task.completion, turn_end: task.turn_end }, turn_end: event }; + } + } else if (opportunity && event.type === "assistant.turn_end" && typeof event.id === "string" && event.id + && assistant?.data.phase === "final_answer" + && typeof assistant.data.turnId === "string" && assistant.data.turnId + && data?.turnId === assistant.data.turnId + && typeof assistant.data.content === "string" && assistant.data.content.trim() + && (assistant.data.toolRequests === undefined + || (Array.isArray(assistant.data.toolRequests) && assistant.data.toolRequests.length === 0))) { + return { kind: "assistant_final", message: assistant, turn_end: event }; } else if (event.type === "session.idle") { if (!data || typeof event.id !== "string" || !event.id || (data.aborted !== undefined && data.aborted !== false)) { return { error: new Error("Native session idle was aborted or malformed") }; } + if (opportunity) { + return { error: new Error("Native opportunity idle lacked a matching turn end") }; + } if (assistant && typeof assistant.data.content === "string" && assistant.data.content.trim() && (assistant.data.toolRequests === undefined || (Array.isArray(assistant.data.toolRequests) && assistant.data.toolRequests.length === 0))) { @@ -610,7 +657,10 @@ export function validateConfig(op, config) { } if (fixture.schema_version === 3 && (fixture.completed_objective || fixture.active_objective) && !(continuationProfile(config) && ["observe", "cleanup"].includes(op))) { - if (!(fixture.runtime_treatment && op === "candidate")) objectiveReadback(goalPath, fixture); + if (!(fixture.runtime_treatment && op === "candidate") + && !(fixture.runtime_treatment === cappedPublicResume && ["observe", "cleanup"].includes(op))) { + objectiveReadback(goalPath, fixture); + } } else if (pathExists(goalPath) && !(continuationProfile(config) && ["observe", "cleanup"].includes(op))) { throw new Error(op === "setup" ? "Setup requires an absent native goal" : "Context calibration requires an absent native goal"); @@ -856,6 +906,10 @@ export async function runProbe(op, config) { let operationalHashes; let resumedAt; let pendingOutcome; + let exactRecordedPrompt; + let promptFailure; + let nativeObservationClosed = false; + let completedNativeOpportunity; const selected = config?.profile !== undefined; let projection; let scheduleMapping; @@ -952,39 +1006,55 @@ export async function runProbe(op, config) { const connect = async () => { if (client) throw new Error("Previous native owner has not stopped"); const fixture = historicalProfile(config) ? contextFixture(config) : null; - const normalize = fixture?.runtime_treatment && (op === "setup" || isCandidate); + const normalize = fixture?.runtime_treatment && (op === "setup" || isCandidate + || (fixture.runtime_treatment === cappedPublicResume && !continuation + && ["observe", "cleanup"].includes(op))); let before; let coldStarted; if (normalize) { if (!report.normalized_active) { - report.runtime_treatment = normalizedActiveCanary; - report.provenance = "synthetic-continuation-canary"; + report.runtime_treatment = fixture.runtime_treatment; + report.provenance = fixture.runtime_treatment === cappedPublicResume + ? fixture.provenance : "synthetic-continuation-canary"; report.historical_active_equivalence = false; + if (fixture.runtime_treatment === cappedPublicResume) { + report.runtime_differences = [ + "Cold load pauses the imported active objective with resumed_session and a fresh updatedAt", + "Public autopilot --max-ai-credits 100 resets continuationCount, increments resumeCount, updates origin/time", + "The original source store has no creditLimit; the command adds/resets a fixed 100-credit window with initial usage zero", + "Cumulative turn/token/credit counters remain source-bound and are not charged against that fresh window", + "Returned command prompt is not consumed; only the separately validated original autopilot request may be sent", + ]; + } report.normalized_active = { source_store: fixture.active_objective_store, source_sha256: fixture.active_objective.sha256, transitions: [], completion_eligibility: "Command owner records eligibility; no public readback. Native task_complete remains unproven.", }; normalizedExpected = fixture.active_objective_store; - if (isCandidate) { - if (!continuation || entry.purpose !== "setup-canary") { + if (isCandidate || op === "observe" || op === "cleanup") { + if (isCandidate && (!continuation || !(fixture.runtime_treatment === cappedPublicResume + ? ["setup-canary", "adapted-public-resume"].includes(entry.purpose) : entry.purpose === "setup-canary"))) { throw new Error("Normalized active treatment requires setup-canary continuation"); } - const setupPath = join(dirname(config.context_fixture_file), "setup.json"); + const previousOperation = op === "cleanup" ? "observe" : "setup"; + const setupPath = join(dirname(config.context_fixture_file), `${previousOperation}.json`); requirePath(setupPath, "file"); const stat = lstatSync(setupPath); const setupBytes = readFileSync(setupPath); const setup = parseDataJson(setupBytes.toString("utf8")); if (stat.uid !== userInfo().uid || stat.nlink !== 1 || (stat.mode & 0o077) - || setup.status !== "SETUP_OBSERVED" || setup.session_id !== config.session_id - || setup.runtime_treatment !== normalizedActiveCanary + || setup.status !== (previousOperation === "setup" ? "SETUP_OBSERVED" : "OBSERVED") + || setup.session_id !== config.session_id + || setup.runtime_treatment !== fixture.runtime_treatment || setup.normalized_active?.source_sha256 !== fixture.active_objective.sha256 || !isDeepStrictEqual(setup.normalized_active?.source_store, fixture.active_objective_store) - || setup.normalized_active?.transitions?.length !== 2) { + || setup.normalized_active?.transitions?.length !== (previousOperation === "setup" ? 2 : 1)) { throw new Error("Missing matching normalized setup receipt"); } normalizedExpected = setup.normalized_active.transitions.at(-1).resumed.store; - report.normalized_active.setup_receipt_sha256 = hash(setupBytes); + if (previousOperation === "setup") report.normalized_active.setup_receipt_sha256 = hash(setupBytes); + else report.normalized_active.observe_receipt_sha256 = hash(setupBytes); } } before = normalizedReadback(report.goal_path, fixture, normalizedExpected); @@ -1867,22 +1937,58 @@ export async function runProbe(op, config) { let active = true; let resolveOutcome; const outcome = new Promise(resolve => { resolveOutcome = resolve; }); + const boundedNative = entry?.native_opportunity_limit === 2; + if (boundedNative) { + report.native_opportunities = []; + completedNativeOpportunity = () => report.native_opportunities[0]?.kind + || (messageId === undefined + && buffered.filter(event => event.type === "user.message").length === 1 + && continuationCompletion(buffered, + buffered.find(event => event.type === "user.message")?.data?.messageId, true)?.kind); + } + const finishOutcome = terminal => { + if (boundedNative) nativeObservationClosed = true; + resolveOutcome(terminal); + }; + if (exactRecordedPrompt !== undefined) promptFailure = error => finishOutcome({ error }); if (continuation) pendingOutcome = () => { - if (messageId !== undefined) resolveOutcome({ pending: true }); + if (messageId !== undefined) finishOutcome({ pending: true }); }; const consume = event => { if (!active || event.agentId || event.data?.parentToolCallId) return; if (event.type === "session.error") { - resolveOutcome({ error: new Error(event.data?.message ?? "Native root session error") }); + finishOutcome({ error: new Error(event.data?.message ?? "Native root session error") }); } else if (event.type === "session.idle" && event.data?.aborted === true) { - resolveOutcome({ error: new Error("Native session idle was aborted") }); + finishOutcome({ error: new Error("Native session idle was aborted") }); } else if (messageId === undefined) { buffered.push(event); } else if (continuation) { completionEvents.push(event); - const terminal = event.type === "session.idle" - ? continuationCompletion(completionEvents, messageId) : undefined; - if (terminal) resolveOutcome(terminal); + if (boundedNative) { + const users = completionEvents.filter(item => item.type === "user.message"); + if (users.length > 2 || (users.length && users[0].data?.messageId !== messageId)) { + finishOutcome({ error: new Error("Unexpected native continuation root event") }); + return; + } + const index = report.native_opportunities.length; + const user = users[index]; + // Native autopilot retains the initiating send's origin across interactions. + const terminal = user && continuationCompletion( + completionEvents.slice(completionEvents.indexOf(user)), user.data?.messageId, true, messageId); + if (terminal?.error) finishOutcome(terminal); + else if (terminal) { + report.native_opportunities.push({ message_id: user.data.messageId, kind: terminal.kind, + message_event_id: terminal.message.id, turn_end_event_id: terminal.turn_end?.id }); + if (terminal.kind === "task_complete" || report.native_opportunities.length === 2) { + report.native_observation_stop = terminal.kind === "task_complete" ? "task_complete" : "opportunity_limit"; + finishOutcome(terminal); + } + } + } else { + const terminal = event.type === "session.idle" + ? continuationCompletion(completionEvents, messageId) : undefined; + if (terminal) resolveOutcome(terminal); + } } else if (event.type === "assistant.message") { matchingMessage = event.data?.originatingMessageId === messageId && typeof event.id === "string" && typeof event.data.content === "string" @@ -1902,12 +2008,14 @@ export async function runProbe(op, config) { const modelStarted = performance.now(); report.model_timing_source = "Owned SDK send through correlated native session.idle, not provider HTTP timing"; let deadlineTimer; + let abortedAtBoundary = false; try { try { const deadline = new Promise(resolve => { - deadlineTimer = setTimeout(() => resolve({ - error: new Error(`${report.timeout_ms}-millisecond SDK send/completion deadline exceeded`), - }), report.timeout_ms); + deadlineTimer = setTimeout(() => { + if (boundedNative) nativeObservationClosed = true; + resolve({ error: new Error(`${report.timeout_ms}-millisecond SDK send/completion deadline exceeded`) }); + }, report.timeout_ms); }); const sendRequest = report.recorded_autopilot_send ? { ...report.recorded_autopilot_send, mode: "immediate", wait: false } @@ -1921,14 +2029,14 @@ export async function runProbe(op, config) { report.model_message_id = id; if (!active) return; messageId = id; - if (continuation && report.pending_input?.length) resolveOutcome({ pending: true }); + if (continuation && report.pending_input?.length) finishOutcome({ pending: true }); for (const event of buffered) consume(event); buffered.length = 0; - }).catch(error => resolveOutcome({ error })); + }).catch(error => finishOutcome({ error })); const terminal = await Promise.race([outcome, deadline]); if (terminal.error) throw terminal.error; report.model_result = terminal.message; - report.model_completion_event = terminal.idle; + report.model_completion_event = terminal.idle ?? terminal.turn_end; if (continuation && !terminal.pending) { report.model_completion_kind = terminal.kind; if (terminal.task_complete) report.model_task_completion = terminal.task_complete; @@ -1936,13 +2044,19 @@ export async function runProbe(op, config) { if (terminal.pending) { report.pending_input_observed = true; await call("session.abort", undefined, () => session.abort()); + abortedAtBoundary = boundedNative; + } else if (boundedNative && terminal.kind !== "task_complete") { + await call("session.abort", undefined, () => session.abort()); + abortedAtBoundary = true; } } catch (error) { fail(error); throw error; } finally { active = false; + nativeObservationClosed = true; pendingOutcome = undefined; + promptFailure = undefined; clearTimeout(deadlineTimer); try { stopCompletion(); } catch (error) { fail(error); } report.model_completed_at = new Date().toISOString(); @@ -1963,9 +2077,17 @@ export async function runProbe(op, config) { throw new Error("Frozen continuation configuration changed"); } report.prefix_after = await readEvents(); + if (exactRecordedPrompt !== undefined + && (report.recorded_prompt_transform.invocations !== + (boundedNative ? 1 + report.recorded_prompt_transform.native_passthrough.length : 1) + || !report.recorded_prompt_transform.input_verified + || report.recorded_prompt_transform.replacement_sha256 !== hash(exactRecordedPrompt))) { + throw new Error("Recorded transformed prompt hook did not verify exactly one replacement"); + } const correspondence = continuationCorrespondence( report.prefix_before, report.prefix_after, candidateEvents.slice(report.model_event_start_index), - report.model_message_id, report.bound_raw_trigger, report.recorded_autopilot_send); + report.model_message_id, report.bound_raw_trigger, report.recorded_autopilot_send, exactRecordedPrompt, + boundedNative ? report.recorded_prompt_transform.native_passthrough : undefined); Object.assign(report.checks, correspondence); report.primary_system_messages = correspondence.primary_system_messages; delete report.checks.primary_system_messages; @@ -1977,8 +2099,13 @@ export async function runProbe(op, config) { && store.current.id === normalizedExpected.current.id && store.current.objective === normalizedExpected.current.objective && isDeepStrictEqual(report.final_state.goal?.state, objectiveState(store)); - if (!report.checks.normalized_native_completion) { - throw new Error("Normalized canary requires genuine task_complete and matching completed native/persisted objective"); + if (entry.purpose === "setup-canary" && (boundedNative + ? report.native_opportunities.length !== 2 + || (report.model_completion_kind === "task_complete" && !report.checks.normalized_native_completion) + : !report.checks.normalized_native_completion)) { + throw new Error(boundedNative + ? "Two-opportunity canary requires two completed native opportunities and consistent task-complete state" + : "Normalized canary requires genuine task_complete and matching completed native/persisted objective"); } } report.tool_effects = candidateEvents.slice(report.model_event_start_index) @@ -2005,7 +2132,9 @@ export async function runProbe(op, config) { report.status = report.pending_input_observed ? "HUMAN_INPUT_PENDING" : "CANDIDATE_OBSERVED"; } catch (error) { fail(error); - try { await call("session.abort", undefined, () => session.abort()); } catch (abortError) { fail(abortError); } + if (!abortedAtBoundary) { + try { await call("session.abort", undefined, () => session.abort()); } catch (abortError) { fail(abortError); } + } } }; try { @@ -2154,6 +2283,62 @@ export async function runProbe(op, config) { onElicitationRequest: request => recordPending("elicitation", request), }; delete candidateOptions.availableTools; + const fixture = contextFixture(config); + if (fixture.runtime_treatment === cappedPublicResume && fixture.autopilot_send) { + exactRecordedPrompt = fixture.canonical.chat_messages.at(-1).content; + report.recorded_prompt_transform = { + surface: "resumeOptions.hooks.onUserPromptTransformed", + source_event_id: fixture.source_binding.cutoff_event_id, + canonical_sha256: fixture.canonical_sha256, + expected_sha256: hash(exactRecordedPrompt), invocations: 0, + ...(entry.native_opportunity_limit === 2 ? { native_passthrough: [], held_after_boundary: 0 } : {}), + }; + candidateOptions.hooks = { + ...candidateOptions.hooks, + onUserPromptTransformed(input, invocation) { + const observed = report.recorded_prompt_transform; + if (entry.native_opportunity_limit === 2 + && (nativeObservationClosed || observed.invocations >= 2 + || completedNativeOpportunity?.() === "task_complete")) { + observed.held_after_boundary++; + return new Promise(() => {}); + } + observed.invocations++; + if (entry.native_opportunity_limit === 2 && observed.invocations === 2) { + if (completedNativeOpportunity?.() !== "assistant_final" || report.model_send_calls !== 1 + || input?.sessionId !== config.session_id || invocation?.sessionId !== config.session_id + || typeof input.prompt !== "string" + || typeof input.transformedPrompt !== "string" || !input.transformedPrompt) { + const error = new Error("Native continuation requires a completed first native opportunity"); + fail(error); + promptFailure?.(error); + return new Promise(() => {}); + } + observed.native_passthrough.push({ + prompt_sha256: hash(input.prompt), transformed_sha256: hash(input.transformedPrompt), + }); + return {}; + } + const wrapper = typeof input?.transformedPrompt === "string" + ? input.transformedPrompt.match(/([0-9T:.+Z-]+)<\/current_datetime>\n\n/) : null; + if (observed.invocations !== 1 || report.model_send_calls !== 1 + || input?.sessionId !== config.session_id || invocation?.sessionId !== config.session_id + || input?.prompt !== fixture.autopilot_send.prompt + || !wrapper || !Number.isFinite(Date.parse(wrapper[1])) + || !input.transformedPrompt.includes(fixture.autopilot_send.prompt)) { + const error = new Error("Recorded transformed prompt hook input differs from the once-only source-bound send"); + fail(error); + promptFailure?.(error); + // The pinned SDK swallows throws. Hold this hook while the existing abort/shutdown path runs. + return new Promise(() => {}); + } + observed.input_verified = true; + observed.input_sha256 = hash(input.transformedPrompt); + observed.replacement_sha256 = hash(exactRecordedPrompt); + return { modifiedTransformedPrompt: exactRecordedPrompt }; + }, + }; + } } report.connection.mode = "copilot-cli"; report.connection.workingDirectory = config.cwd; diff --git a/skills/skill-evaluation/scripts/native_workflow_probe.test.mjs b/skills/skill-evaluation/scripts/native_workflow_probe.test.mjs index 8e3d215..e063d53 100644 --- a/skills/skill-evaluation/scripts/native_workflow_probe.test.mjs +++ b/skills/skill-evaluation/scripts/native_workflow_probe.test.mjs @@ -72,6 +72,11 @@ export class CopilotClient { let goalStore; try { goalStore = JSON.parse(readFileSync(goalPath, "utf8")); current = goalStore.current; } catch {} const persistGoal = () => writeFileSync(goalPath, JSON.stringify(goalStore)); + state.completeGoal = () => { + current.status = "completed"; + current.completionSummary = "Synthetic completion"; + persistGoal(); + }; if (current && state.pauseOnCold && current.status === "active") { Object.assign(current, { status: "paused", pauseReason: "resumed_session", updatedAt: new Date().toISOString() }); @@ -96,22 +101,36 @@ export class CopilotClient { state.sentId = id; state.sendReady?.(); if (state.sendFails) throw new Error("Test send transport failure"); - if (state.sendHook) { - const messageId = state.sendHook(id, emit); - if (state.completeOnSend) { - current.status = "completed"; - current.completionSummary = "Synthetic completion"; - persistGoal(); + const finish = () => { + if (state.sendHook) { + const messageId = state.sendHook(id, emit); + if (state.completeOnSend) { + current.status = "completed"; + current.completionSummary = "Synthetic completion"; + persistGoal(); + } + return messageId; } - return messageId; + emit("assistant.turn_start", { model: options.model }); + emit("assistant.usage", { apiCallId: "test-api-1", model: options.model }); + emit("assistant.usage", { apiCallId: "test-api-2", model: options.model }); + emit("assistant.message", { content: "Test SDK response", originatingMessageId: id }); + emit("assistant.idle", {}); + emit("session.idle", { mode: "autopilot" }); + return id; + }; + if (options.hooks?.onUserPromptTransformed) { + void Promise.resolve().then(async () => { + const input = { sessionId: session.sessionId, + prompt: message.prompt, + transformedPrompt: "2026-02-02T00:00:00.000Z\\n\\n" + message.prompt }; + const output = await options.hooks.onUserPromptTransformed(input, { sessionId: session.sessionId }); + state.transformedPrompt = output?.modifiedTransformedPrompt ?? input.transformedPrompt; + finish(); + }).catch(error => emit("session.error", { message: error.message })); + return id; } - emit("assistant.turn_start", { model: options.model }); - emit("assistant.usage", { apiCallId: "test-api-1", model: options.model }); - emit("assistant.usage", { apiCallId: "test-api-2", model: options.model }); - emit("assistant.message", { content: "Test SDK response", originatingMessageId: id }); - emit("assistant.idle", {}); - emit("session.idle", { mode: "autopilot" }); - return id; + return finish(); }, rpc: { commands: { invoke: async request => { @@ -864,7 +883,8 @@ async function continuationFixture(t, { loadedCopies = false, operational = fals const persist = (type, data) => state.events.push(emit(type, data)); persist("user.message", { ...(autopilot ? { content: "", source: "autopilot", agentMode: "autopilot", - transformedContent: `2026-02-02T00:00:00.000Z\n\n${owned.request.prompt}`, + transformedContent: state.transformedPrompt + ?? `2026-02-02T00:00:00.000Z\n\n${owned.request.prompt}`, } : { content: "Synthetic trigger" }), messageId: id, interactionId: id }); emit("system.message", { interactionId: id, content: input.cli_preamble @@ -1293,6 +1313,168 @@ async function normalizedCanaryFixture(t) { return { ...owned, setupConfig }; } +async function twoOpportunityFixture(t) { + const owned = await normalizedCanaryFixture(t); + const { config, store, source, goal, locator, input, save, state } = owned; + Object.assign(store.current, { continuationCount: 10, turnCount: 837, tokenCount: 128149341, + creditCountNanoAiu: "3137066880000", resumeCount: 0 }); + delete store.current.creditLimit; + const bytes = JSON.stringify(store); + writeFileSync(source, bytes); + writeFileSync(goal, bytes); + locator.active_objective.sha256 = createHash("sha256").update(bytes).digest("hex"); + locator.runtime_treatment = "capped-public-resume-v1"; + locator.provenance = "synthetic-continuation-canary"; + writeFileSync(config.context_fixture_file, JSON.stringify(locator)); + input.native_opportunity_limit = 2; + save(); + state.completeViaTool = false; + state.completeOnSend = false; + assert.equal((await runProbe("setup", owned.setupConfig)).status, "SETUP_OBSERVED"); + return owned; +} + +test("bounded native observation binds its selector and rejects invalid or uncapped choices before send", async t => { + const { config, root, input, save, state, locator } = await twoOpportunityFixture(t); + const selected = continuationEntry(config); + delete input.native_opportunity_limit; + save(); + const original = continuationEntry(config); + assert.notEqual(selected.configuration_sha256, original.configuration_sha256); + assert.equal(selected.binding.native_opportunity_limit, 2); + assert.equal(original.binding.native_opportunity_limit, undefined); + for (const value of [true, 1, 3, 2.5, "2", null]) { + config.output = join(root, `invalid-${randomUUID()}.json`); + input.native_opportunity_limit = value; + save(); + const result = await runProbe("candidate", config); + assert.equal(result.status, "PREMODEL_BLOCKED"); + assert.match(result.error.message, /native_opportunity_limit/); + } + input.native_opportunity_limit = 2; + locator.runtime_treatment = "public-command-normalized-active-v1"; + delete locator.provenance; + writeFileSync(config.context_fixture_file, JSON.stringify(locator)); + save(); + assert.throws(() => continuationEntry(config), /requires capped/); + assert.equal(state.sendCalls, 0); +}); + +for (const scenario of ["second-final", "empty-native-raw", "first-task", "second-task", "protected-input", "native-error", + "aborted", "deadline", "early-prompt", "substituted-root", "substituted-origin", "unmatched-turn", + "missing-end-idle", "unmatched-end-idle", "nonfinal-idle"]) { + test(`bounded native observation owning capture: ${scenario}`, async t => { + const { config, input, save, state, goal } = await twoOpportunityFixture(t); + if (["deadline", "substituted-origin", "unmatched-turn"].includes(scenario)) { input.timeout_ms = 25; save(); } + if (scenario === "first-task") { + state.completeViaTool = true; + state.completeOnSend = true; + } + const first = state.sendHook; + state.forwardedNativePrompts = 0; + state.thirdModels = 0; + state.sendHook = (id, emit) => { + const persist = (type, data) => state.events.push(emit(type, data)); + first(id, (type, data, extra) => { + if (type === "session.idle") return { id: randomUUID(), type, data }; + if (type === "assistant.message" && scenario !== "first-task") { + Object.assign(data, { turnId: "first", phase: "final_answer", toolRequests: [] }); + } + return emit(type, data, extra); + }); + state.events = state.events.filter(event => event.type !== "session.idle"); + if (scenario === "deadline" || scenario === "first-task") return id; + if (scenario !== "early-prompt") persist("assistant.turn_end", { turnId: "first" }); + const hook = state.resumes.at(-1).hooks.onUserPromptTransformed; + const next = { sessionId: config.session_id, prompt: "Synthetic native continuation", + transformedPrompt: "2026-02-02T00:00:01.000Z\n\nSynthetic native continuation" }; + if (scenario === "empty-native-raw") next.prompt = ""; + void Promise.resolve(hook(next, { sessionId: config.session_id })).then(output => { + state.forwardedNativePrompts++; + assert.equal(output?.modifiedTransformedPrompt, undefined); + const second = randomUUID(); + persist("user.message", { content: "", source: "autopilot", agentMode: "autopilot", + transformedContent: scenario === "substituted-root" ? "Substituted native input" : next.transformedPrompt, + messageId: second, interactionId: second }); + emit("system.message", { interactionId: second, content: input.cli_preamble + + "\nYou are an interactive tool that helps users with software engineering tasks." }); + if (scenario === "protected-input") { + void state.resumes.at(-1).onUserInputRequest({ question: "Synthetic protected input" }); + } else if (scenario === "native-error") { + emit("session.error", { message: "Synthetic native continuation failure" }); + } else if (scenario === "aborted") { + emit("session.idle", { aborted: true }); + } else if (scenario === "second-task") { + const request = { name: "task_complete", toolCallId: "second-task", arguments: { summary: "Synthetic completion" } }; + persist("assistant.message", { content: null, originatingMessageId: id, interactionId: second, + turnId: "second", toolRequests: [request] }); + persist("tool.execution_start", { toolName: request.name, toolCallId: request.toolCallId, + arguments: request.arguments, turnId: "second" }); + state.completeGoal(); + persist("tool.execution_complete", { toolCallId: request.toolCallId, success: true, + interactionId: second, turnId: "second", result: { content: "Synthetic completion" } }); + persist("assistant.turn_end", { turnId: "second" }); + } else { + persist("assistant.message", { content: "Synthetic second completed opportunity.", + originatingMessageId: scenario === "substituted-origin" ? randomUUID() : id, + interactionId: second, turnId: "second", + phase: scenario === "nonfinal-idle" ? "commentary" : "final_answer", toolRequests: [] }); + if (scenario !== "missing-end-idle") { + persist("assistant.turn_end", { turnId: ["unmatched-turn", "unmatched-end-idle"].includes(scenario) + ? "unrelated" : "second" }); + } + if (scenario.endsWith("-idle")) persist("session.idle", {}); + void Promise.resolve(hook(next, { sessionId: config.session_id })).then(() => { state.thirdModels++; }); + } + }); + return id; + }; + const result = await runProbe("candidate", config); + const expected = ["second-final", "empty-native-raw", "second-task"].includes(scenario) ? "CANDIDATE_OBSERVED" + : scenario === "protected-input" ? "HUMAN_INPUT_PENDING" : "FAILED"; + assert.equal(result.status, expected, result.error?.message ?? result.status); + assert.equal(state.sendCalls, 1); + assert.equal(state.thirdModels, 0); + assert.equal(JSON.parse(readFileSync(goal)).current.status, + ["first-task", "second-task"].includes(scenario) ? "completed" : "active"); + assert.equal(state.aborts, scenario === "second-task" ? 0 : 1); + if (["second-final", "empty-native-raw"].includes(scenario)) { + assert.equal(result.native_opportunities.length, 2); + assert.equal(result.model_result.data.originatingMessageId, result.model_message_id); + assert.equal(result.native_observation_stop, "opportunity_limit"); + assert.equal(result.model_completion_event.type, "assistant.turn_end"); + assert.equal(result.checks.exactly_once, true); + assert.equal(result.checks.native_continuation_passthrough, true); + assert.equal(result.checks.normalized_native_completion, false); + assert.equal(result.recorded_prompt_transform.held_after_boundary, 1); + assert.equal(result.continuation_receipt.status, "CONTINUATION_CANARY_OBSERVED"); + } else if (scenario === "early-prompt") { + assert.equal(state.forwardedNativePrompts, 0); + assert.match(result.error.message, /completed first native opportunity/); + } else if (scenario === "substituted-root") { + assert.match(result.error.message, /native continuation/); + } else if (["deadline", "substituted-origin", "unmatched-turn"].includes(scenario)) { + assert.match(result.error.message, /deadline/); + } else if (scenario.endsWith("-idle")) { + assert.equal(result.native_opportunities.length, 1); + assert.match(result.error.message, /matching turn end/); + } else if (scenario === "native-error") { + assert.match(result.error.message, /Synthetic native continuation failure/); + } else if (scenario === "aborted") { + assert.match(result.error.message, /Native session idle was aborted/); + } else if (scenario === "first-task") { + assert.equal(state.forwardedNativePrompts, 0); + assert.equal(result.native_opportunities.length, 1); + assert.equal(result.model_completion_kind, "task_complete"); + assert.match(result.error.message, /Two-opportunity canary/); + } else if (scenario === "second-task") { + assert.equal(result.native_opportunities.length, 2); + assert.equal(result.model_completion_kind, "task_complete"); + assert.equal(result.checks.normalized_native_completion, true); + } + }); +} + test("normalized canary witnesses both cold pauses and resumes without sending the command prompt", async t => { const { config, setupConfig, state, store, goal, request, input, save } = await normalizedCanaryFixture(t); const setup = await runProbe("setup", setupConfig); diff --git a/skills/skill-evaluation/scripts/session_restore.py b/skills/skill-evaluation/scripts/session_restore.py index 74ef3d7..f318d8b 100644 --- a/skills/skill-evaluation/scripts/session_restore.py +++ b/skills/skill-evaluation/scripts/session_restore.py @@ -37,6 +37,7 @@ def active_objective_state(store: dict) -> dict: NORMALIZED_ACTIVE_CANARY = "public-command-normalized-active-v1" +CAPPED_PUBLIC_RESUME = "capped-public-resume-v1" def normalized_active_canary(store: dict, source: str) -> None: @@ -48,6 +49,18 @@ def normalized_active_canary(store: dict, source: str) -> None: raise ValueError("Normalized active treatment requires the capped synthetic canary") +def capped_public_resume(store: dict, source: str, provenance: str) -> None: + active_objective_state(store) + synthetic = provenance == "synthetic-continuation-canary" + if (provenance not in ("synthetic-continuation-canary", "source-bound-adapted-public-resume") + or not isinstance(source, str) or not source.startswith("/") + or not source.endswith("/events.jsonl") + or (synthetic and not re.fullmatch(r"/synthetic/[0-9a-f-]{36}/events\.jsonl", source)) + or (not synthetic and source.startswith("/synthetic/")) + or "creditLimit" in store["current"]): + raise ValueError("Adapted capped public resume requires an initially uncapped active objective and matching provenance") + + def _objective_state(store: dict, status: str) -> dict: required = {"id", "objective", "status", "autopilotOrigin", "continuationCount", "turnCount", "tokenCount", "creditCountNanoAiu", "resumeCount", "createdAt", "updatedAt"} diff --git a/skills/skill-evaluation/scripts/skill_eval.py b/skills/skill-evaluation/scripts/skill_eval.py index abf420e..9b9d57e 100644 --- a/skills/skill-evaluation/scripts/skill_eval.py +++ b/skills/skill-evaluation/scripts/skill_eval.py @@ -2494,6 +2494,9 @@ def append(kind, data): } if "runtime_treatment" in historical: locator["runtime_treatment"] = historical["runtime_treatment"] + from session_restore import CAPPED_PUBLIC_RESUME + if historical["runtime_treatment"] == CAPPED_PUBLIC_RESUME: + locator["provenance"] = historical["provenance"] if objective_key in historical: locator[objective_key] = { "path": str(owner / f"{objective_key.replace('_', '-')}.json"), @@ -2834,23 +2837,25 @@ def _validate_recorded_context_fixture(fixture: dict, root: Path | None) -> dict from session_restore import ( active_objective_state, compile_task_snapshot, completed_objective_state, permission_rules, recorded_autopilot_request, recorded_native_payloads, - NORMALIZED_ACTIVE_CANARY, normalized_active_canary, + NORMALIZED_ACTIVE_CANARY, normalized_active_canary, CAPPED_PUBLIC_RESUME, capped_public_resume, ) keys = {"schema_version", "provenance", "recorded_context", "restoration", "trigger", "consumer", "snapshot", "required_tools", "limitations", "personal_instructions", "candidate_permissions"} autopilot = "recorded_autopilot_request" in fixture normalized = "runtime_treatment" in fixture + capped = fixture.get("runtime_treatment") == CAPPED_PUBLIC_RESUME + provenances = (("synthetic-continuation-canary", "source-bound-adapted-public-resume") if capped else + ("synthetic-continuation-canary",) if normalized else ("source-bound-historical",)) if (set(fixture) - {"completed_objective", "active_objective", "recorded_autopilot_request", "runtime_treatment"} != keys or ("active_objective" in fixture) != autopilot or (autopilot and "completed_objective" in fixture) or (normalized and (not autopilot - or fixture["runtime_treatment"] != NORMALIZED_ACTIVE_CANARY)) + or fixture["runtime_treatment"] not in (NORMALIZED_ACTIVE_CANARY, CAPPED_PUBLIC_RESUME))) or type(fixture["schema_version"]) is not int or fixture["schema_version"] != 3 - or fixture["provenance"] != ( - "synthetic-continuation-canary" if normalized else "source-bound-historical") + or fixture["provenance"] not in provenances or root is None or not isinstance(fixture["personal_instructions"], str) or not fixture["personal_instructions"].strip() @@ -2959,7 +2964,10 @@ def _validate_recorded_context_fixture(fixture: dict, root: Path | None) -> dict read_owner_json(request_path), event, trigger, store) objective["recorded_autopilot_request_bytes"] = request_bytes if normalized: - normalized_active_canary(store, context["source"]) + if capped: + capped_public_resume(store, context["source"], fixture["provenance"]) + else: + normalized_active_canary(store, context["source"]) permission_rules(fixture["candidate_permissions"]) return {**fixture, "snapshot": compile_task_snapshot(fixture["snapshot"]), "canonical": context, "canonical_bytes": source_bytes, @@ -3118,13 +3126,21 @@ def _run_native_context_calibration( raise ValueError("Context fixture requires an owner-only unaliased single-link file") fixture = read_owner_json(fixture_file) normalized = isinstance(fixture, dict) and "runtime_treatment" in fixture + from session_restore import CAPPED_PUBLIC_RESUME + capped = isinstance(fixture, dict) and fixture.get("runtime_treatment") == CAPPED_PUBLIC_RESUME historical = isinstance(fixture, dict) and ( fixture.get("provenance") == "source-bound-historical" or normalized) if normalized: - if continuation_file is None or source_preparation_only or not model_send: + zero_model_capped_preparation = ( + capped and fixture.get("provenance") == "source-bound-adapted-public-resume" + and source_preparation_only and continuation_file is None and not model_send) + if (not zero_model_capped_preparation + and (continuation_file is None or source_preparation_only or not model_send)): raise ValueError("Normalized active treatment requires the explicit continuation canary") result["runtime_treatment"] = fixture["runtime_treatment"] result["historical_active_equivalence"] = False + if capped: + result["provenance"] = fixture["provenance"] calibration_session_id = None if source_preparation_only: if not historical or model_send: @@ -3138,7 +3154,9 @@ def _run_native_context_calibration( or any(path.is_symlink() for path in (personal_skill_dir, *personal_skill_dir.parents))): raise ValueError("Personal skill discovery requires an unaliased historical definition directory") if historical: - result["claim"] = ("Synthetic normalized-active infrastructure only; historical equivalence unproved" + result["claim"] = ("Adapted source-bound capped public-resume setup only; historical active equivalence false" + if capped and fixture["provenance"] == "source-bound-adapted-public-resume" else + "Synthetic normalized-active infrastructure only; historical equivalence unproved" if normalized else "Historical setup observations only; behavioral admission and skill improvement unproved") if model_send and continuation_file is None: @@ -3372,6 +3390,12 @@ def execute_import(command): result["claim"] = ("Synthetic continuation capability observation; historical reproduction and skill efficacy unproved" if result.get("candidate", {}).get("continuation_entry", {}).get("purpose") == "setup-canary" else "Historical continuation observation; complete baseline fidelity and skill efficacy unproved") + if result.get("runtime_treatment") == "capped-public-resume-v1": + result["claim"] = ( + "Adapted source-bound capped public-resume observation; historical active equivalence false; " + "behavioral admission and skill efficacy unproved" + if result.get("provenance") == "source-bound-adapted-public-resume" else + "Synthetic capped public-resume capability observation only; historical behavior unproved") write_json(run_root / "context-result.json", result) return result