diff --git a/skills/skill-evaluation/scripts/native_workflow.py b/skills/skill-evaluation/scripts/native_workflow.py index 7e5ea95..624031f 100644 --- a/skills/skill-evaluation/scripts/native_workflow.py +++ b/skills/skill-evaluation/scripts/native_workflow.py @@ -165,13 +165,17 @@ def owned(source): "schema_version", "purpose", "running_in_interactive_mode", "session_capabilities", "cli_preamble", "executable_path", "locale", "timeout_ms", "package_root", "workflow_commit", "workflow_sha256", "canary_receipt", "canary_checks", - } | ({"skill_char_budget"} if isinstance(value, dict) and "skill_char_budget" in value else set()), + } | ({key for key in ("skill_char_budget", "native_opportunity_limit") if key in value} + if isinstance(value, dict) else set()), "historical continuation") + if "native_opportunity_limit" in value and ( + type(value["native_opportunity_limit"]) is not int or value["native_opportunity_limit"] != 2): + raise ValueError("native_opportunity_limit must be the integer 2 when explicitly selected") if "skill_char_budget" in value and ( type(value["skill_char_budget"]) is not int or not 1 <= value["skill_char_budget"] <= 4294967295): raise ValueError("skill_char_budget must be an integer in 1..4294967295") if (type(value["schema_version"]) is not int or value["schema_version"] != 1 - or value["purpose"] not in ("setup-canary", "historical") + or value["purpose"] not in ("setup-canary", "historical", "adapted-public-resume") or value["running_in_interactive_mode"] is not True or value["session_capabilities"] != CLI_CONTINUATION_CAPABILITIES or value["cli_preamble"] != CLI_CONTINUATION_PREAMBLE @@ -220,15 +224,23 @@ def owned(source): } if "skill_char_budget" in value: binding["interactive"]["skill_char_budget"] = value["skill_char_budget"] + capped = False if "context_fixture_file" in config: fixture = read_owner_json(owned(config["context_fixture_file"])) if "runtime_treatment" in fixture: - from session_restore import NORMALIZED_ACTIVE_CANARY - if (fixture["runtime_treatment"] != NORMALIZED_ACTIVE_CANARY - or value["purpose"] != "setup-canary" + from session_restore import NORMALIZED_ACTIVE_CANARY, CAPPED_PUBLIC_RESUME + capped = fixture["runtime_treatment"] == CAPPED_PUBLIC_RESUME + if ((not capped and (fixture["runtime_treatment"] != NORMALIZED_ACTIVE_CANARY + or value["purpose"] != "setup-canary")) + or (capped and ( + (value["purpose"], fixture.get("provenance")) not in ( + ("setup-canary", "synthetic-continuation-canary"), + ("adapted-public-resume", "source-bound-adapted-public-resume")))) or "recorded_autopilot_request" not in fixture): - raise ValueError("Normalized active treatment is setup-canary only") - binding["runtime_treatment"] = NORMALIZED_ACTIVE_CANARY + raise ValueError( + "Adapted capped continuation requires matching synthetic setup-canary or source-bound adapted-public-resume provenance" + if capped else "Normalized active treatment is setup-canary only") + binding["runtime_treatment"] = fixture["runtime_treatment"] if "recorded_autopilot_request" in fixture: if "active_objective" not in fixture or "completed_objective" in fixture: raise ValueError("Recorded autopilot continuation requires its active objective") @@ -237,6 +249,12 @@ def owned(source): "agentMode": "autopilot", "objective_status": "active", "send_surface": "session.rpc.send", "wait": False, } + if value["purpose"] == "adapted-public-resume" and not capped: + raise ValueError("Adapted public-resume purpose requires the capped treatment and source-bound provenance") + if "native_opportunity_limit" in value: + if not capped: + raise ValueError("native_opportunity_limit requires capped source-bound autopilot continuation") + binding["native_opportunity_limit"] = value["native_opportunity_limit"] if "operational_file" in config: binding["operational"] = operational_input(Path(config["operational_file"]), config)["binding"] configuration_sha256 = hashlib.sha256(json.dumps( diff --git a/skills/skill-evaluation/scripts/native_workflow_probe.mjs b/skills/skill-evaluation/scripts/native_workflow_probe.mjs index 8e37571..aee73cf 100644 --- a/skills/skill-evaluation/scripts/native_workflow_probe.mjs +++ b/skills/skill-evaluation/scripts/native_workflow_probe.mjs @@ -173,6 +173,7 @@ function profileState(config) { } const normalizedActiveCanary = "public-command-normalized-active-v1"; +const cappedPublicResume = "capped-public-resume-v1"; function contextFixture(config) { const path = config.context_fixture_file; @@ -196,8 +197,10 @@ function contextFixture(config) { if (active) keys.push("active_objective"); if (autopilot) keys.push("recorded_autopilot_request"); if (normalized) keys.push("runtime_treatment"); + if (fixture.runtime_treatment === cappedPublicResume) keys.push("provenance"); if (!isDeepStrictEqual(Object.keys(fixture).sort(), keys.sort()) - || (normalized && (!active || fixture.runtime_treatment !== normalizedActiveCanary)) + || (normalized && (!active + || ![normalizedActiveCanary, cappedPublicResume].includes(fixture.runtime_treatment))) || active !== autopilot || (active && completed) || [active && fixture.active_objective, autopilot && fixture.recorded_autopilot_request] .filter(value => value !== false).some(value => !value @@ -240,12 +243,23 @@ function contextFixture(config) { const completedStore = completed ? parseDataJson(readFileSync(fixture.completed_objective.path, "utf8")) : null; const activeStore = active ? parseDataJson(readFileSync(fixture.active_objective.path, "utf8")) : null; - if (normalized && (!/^\/synthetic\/[0-9a-f-]{36}\/events\.jsonl$/.test(canonical.source) + if (fixture.runtime_treatment === normalizedActiveCanary + && (!/^\/synthetic\/[0-9a-f-]{36}\/events\.jsonl$/.test(canonical.source) || activeStore.current.continuationCount !== 0 || !isDeepStrictEqual(activeStore.current.creditLimit, { credits: 100, creditsUsed: 0, creditsUsedNanoAiu: "0" }))) { throw new Error("Normalized active treatment requires the capped synthetic canary"); } + if (fixture.runtime_treatment === cappedPublicResume + && (!["synthetic-continuation-canary", "source-bound-adapted-public-resume"].includes(fixture.provenance) + || typeof canonical.source !== "string" || !canonical.source.startsWith("/") + || !canonical.source.endsWith("/events.jsonl") + || (fixture.provenance === "synthetic-continuation-canary" + ? !/^\/synthetic\/[0-9a-f-]{36}\/events\.jsonl$/.test(canonical.source) + : canonical.source.startsWith("/synthetic/")) + || Object.hasOwn(activeStore.current, "creditLimit"))) { + throw new Error("Adapted capped public resume requires an initially uncapped active objective and matching provenance"); + } const recordedRequest = autopilot ? parseDataJson(readFileSync(fixture.recorded_autopilot_request.path, "utf8")) : null; const ownerRoot = dirname(config.context_fixture_file); @@ -314,7 +328,9 @@ function contextFixture(config) { active_objective_native_state: compiled[3], autopilot_send: compiled[4], limitations: [...fixture.limitations, "Single recorded native event replay only; automatic producer leases, repeated continuations and timers are not reproduced", - "Native runtime regenerates transport current_datetime; the recorded datetime is not sent", + fixture.runtime_treatment === cappedPublicResume + ? "Opted-in candidate replay restores the exact source transformed prompt through the public SDK hook; zero-model setup sends no prompt" + : "Native runtime regenerates transport current_datetime; the recorded datetime is not sent", "Pinned single-send forwarding omits billable; no billing-fidelity claim, actual accounting unchanged", ] } : {}) }; } @@ -460,7 +476,8 @@ function operationalEntry(config) { { encoding: "utf8", env: { PATH: process.env.PATH ?? "" } })); } -export function continuationCorrespondence(before, after, events, messageId, raw, autopilotSend) { +export function continuationCorrespondence(before, after, events, messageId, raw, autopilotSend, exactTransformed, + nativePrompts) { if (after.length < before.length || before.some((event, index) => !isDeepStrictEqual(event, after[index]))) { throw new Error("Historical continuation mutated its native journal prefix"); } @@ -474,13 +491,23 @@ export function continuationCorrespondence(before, after, events, messageId, raw source: "autopilot", agentMode: "autopilot" }) && data?.content === "" && data.source === "autopilot" && data.agentMode === "autopilot" && wrapper && Number.isFinite(Date.parse(wrapper[1])) - && data.transformedContent.slice(wrapper[0].length) === raw; - if (users.length !== 1 || (autopilotSend ? !autopilotMatches + && data.transformedContent.slice(wrapper[0].length) === raw + && (exactTransformed === undefined || data.transformedContent === exactTransformed); + if ((nativePrompts === undefined ? users.length !== 1 : users.length !== nativePrompts.length + 1) + || (autopilotSend ? !autopilotMatches : (users[0].data?.content !== raw || users[0].data?.isAutopilotContinuation || users[0].data?.source)) || users[0].data?.messageId !== messageId || users[0].data?.attachments?.length || typeof users[0].data?.interactionId !== "string" || !users[0].data.interactionId) { throw new Error("Historical continuation requires exactly one unsubstituted root raw user event"); } + if (nativePrompts !== undefined && (nativePrompts.length > 1 || users.slice(1).some((event, index) => { + const later = event.data; + return !autopilotSend || later?.content !== "" || later.source !== "autopilot" || later.agentMode !== "autopilot" + || later.attachments?.length || typeof later.messageId !== "string" || !later.messageId + || later.messageId === messageId || typeof later.interactionId !== "string" || !later.interactionId + || typeof later.transformedContent !== "string" + || hash(later.transformedContent) !== nativePrompts[index].transformed_sha256; + }))) throw new Error("Unbound or substituted native continuation root event"); const primary = events.filter(event => root(event) && event.type === "system.message" && event.data?.interactionId === users[0].data.interactionId); if (!primary.length || primary.some(event => typeof event.data?.content !== "string" @@ -490,11 +517,15 @@ export function continuationCorrespondence(before, after, events, messageId, raw throw new Error("Fresh correlated primary system prompt did not verify interactive CLI identity"); } return { prefix_preserved: true, exactly_once: true, interactive_identity: true, + ...(nativePrompts === undefined ? {} : { native_continuation_passthrough: true }), ...(autopilotSend ? { recorded_autopilot_request: true } : {}), + ...(exactTransformed === undefined ? {} : { + recorded_transformed_content: true, recorded_transformed_sha256: hash(data.transformedContent), + }), primary_system_messages: primary, interaction_id: users[0].data.interactionId }; } -export function continuationCompletion(events, messageId) { +export function continuationCompletion(events, messageId, opportunity = false, originatingMessageId = messageId) { let interactionId; let assistant; let task; @@ -509,7 +540,7 @@ export function continuationCompletion(events, messageId) { && data.interactionId ? data.interactionId : undefined; assistant = task = undefined; } else if (event.type === "assistant.message") { - assistant = interactionId && typeof event.id === "string" && event.id && data?.originatingMessageId === messageId + assistant = interactionId && typeof event.id === "string" && event.id && data?.originatingMessageId === originatingMessageId && data.interactionId === interactionId ? event : undefined; const requests = Array.isArray(assistant?.data.toolRequests) ? assistant.data.toolRequests.filter(request => @@ -528,10 +559,26 @@ export function continuationCompletion(events, messageId) { task.completion = event; } else if (task?.completion && event.type === "assistant.turn_end") { task.turn_end = typeof event.id === "string" && event.id && data?.turnId === assistant.data.turnId ? event : undefined; + if (opportunity && task.turn_end && !task.invalid) { + return { kind: "task_complete", message: assistant, + task_complete: { request: task.request, start: task.start, + completion: task.completion, turn_end: task.turn_end }, turn_end: event }; + } + } else if (opportunity && event.type === "assistant.turn_end" && typeof event.id === "string" && event.id + && assistant?.data.phase === "final_answer" + && typeof assistant.data.turnId === "string" && assistant.data.turnId + && data?.turnId === assistant.data.turnId + && typeof assistant.data.content === "string" && assistant.data.content.trim() + && (assistant.data.toolRequests === undefined + || (Array.isArray(assistant.data.toolRequests) && assistant.data.toolRequests.length === 0))) { + return { kind: "assistant_final", message: assistant, turn_end: event }; } else if (event.type === "session.idle") { if (!data || typeof event.id !== "string" || !event.id || (data.aborted !== undefined && data.aborted !== false)) { return { error: new Error("Native session idle was aborted or malformed") }; } + if (opportunity) { + return { error: new Error("Native opportunity idle lacked a matching turn end") }; + } if (assistant && typeof assistant.data.content === "string" && assistant.data.content.trim() && (assistant.data.toolRequests === undefined || (Array.isArray(assistant.data.toolRequests) && assistant.data.toolRequests.length === 0))) { @@ -610,7 +657,10 @@ export function validateConfig(op, config) { } if (fixture.schema_version === 3 && (fixture.completed_objective || fixture.active_objective) && !(continuationProfile(config) && ["observe", "cleanup"].includes(op))) { - if (!(fixture.runtime_treatment && op === "candidate")) objectiveReadback(goalPath, fixture); + if (!(fixture.runtime_treatment && op === "candidate") + && !(fixture.runtime_treatment === cappedPublicResume && ["observe", "cleanup"].includes(op))) { + objectiveReadback(goalPath, fixture); + } } else if (pathExists(goalPath) && !(continuationProfile(config) && ["observe", "cleanup"].includes(op))) { throw new Error(op === "setup" ? "Setup requires an absent native goal" : "Context calibration requires an absent native goal"); @@ -856,6 +906,10 @@ export async function runProbe(op, config) { let operationalHashes; let resumedAt; let pendingOutcome; + let exactRecordedPrompt; + let promptFailure; + let nativeObservationClosed = false; + let completedNativeOpportunity; const selected = config?.profile !== undefined; let projection; let scheduleMapping; @@ -952,39 +1006,55 @@ export async function runProbe(op, config) { const connect = async () => { if (client) throw new Error("Previous native owner has not stopped"); const fixture = historicalProfile(config) ? contextFixture(config) : null; - const normalize = fixture?.runtime_treatment && (op === "setup" || isCandidate); + const normalize = fixture?.runtime_treatment && (op === "setup" || isCandidate + || (fixture.runtime_treatment === cappedPublicResume && !continuation + && ["observe", "cleanup"].includes(op))); let before; let coldStarted; if (normalize) { if (!report.normalized_active) { - report.runtime_treatment = normalizedActiveCanary; - report.provenance = "synthetic-continuation-canary"; + report.runtime_treatment = fixture.runtime_treatment; + report.provenance = fixture.runtime_treatment === cappedPublicResume + ? fixture.provenance : "synthetic-continuation-canary"; report.historical_active_equivalence = false; + if (fixture.runtime_treatment === cappedPublicResume) { + report.runtime_differences = [ + "Cold load pauses the imported active objective with resumed_session and a fresh updatedAt", + "Public autopilot --max-ai-credits 100 resets continuationCount, increments resumeCount, updates origin/time", + "The original source store has no creditLimit; the command adds/resets a fixed 100-credit window with initial usage zero", + "Cumulative turn/token/credit counters remain source-bound and are not charged against that fresh window", + "Returned command prompt is not consumed; only the separately validated original autopilot request may be sent", + ]; + } report.normalized_active = { source_store: fixture.active_objective_store, source_sha256: fixture.active_objective.sha256, transitions: [], completion_eligibility: "Command owner records eligibility; no public readback. Native task_complete remains unproven.", }; normalizedExpected = fixture.active_objective_store; - if (isCandidate) { - if (!continuation || entry.purpose !== "setup-canary") { + if (isCandidate || op === "observe" || op === "cleanup") { + if (isCandidate && (!continuation || !(fixture.runtime_treatment === cappedPublicResume + ? ["setup-canary", "adapted-public-resume"].includes(entry.purpose) : entry.purpose === "setup-canary"))) { throw new Error("Normalized active treatment requires setup-canary continuation"); } - const setupPath = join(dirname(config.context_fixture_file), "setup.json"); + const previousOperation = op === "cleanup" ? "observe" : "setup"; + const setupPath = join(dirname(config.context_fixture_file), `${previousOperation}.json`); requirePath(setupPath, "file"); const stat = lstatSync(setupPath); const setupBytes = readFileSync(setupPath); const setup = parseDataJson(setupBytes.toString("utf8")); if (stat.uid !== userInfo().uid || stat.nlink !== 1 || (stat.mode & 0o077) - || setup.status !== "SETUP_OBSERVED" || setup.session_id !== config.session_id - || setup.runtime_treatment !== normalizedActiveCanary + || setup.status !== (previousOperation === "setup" ? "SETUP_OBSERVED" : "OBSERVED") + || setup.session_id !== config.session_id + || setup.runtime_treatment !== fixture.runtime_treatment || setup.normalized_active?.source_sha256 !== fixture.active_objective.sha256 || !isDeepStrictEqual(setup.normalized_active?.source_store, fixture.active_objective_store) - || setup.normalized_active?.transitions?.length !== 2) { + || setup.normalized_active?.transitions?.length !== (previousOperation === "setup" ? 2 : 1)) { throw new Error("Missing matching normalized setup receipt"); } normalizedExpected = setup.normalized_active.transitions.at(-1).resumed.store; - report.normalized_active.setup_receipt_sha256 = hash(setupBytes); + if (previousOperation === "setup") report.normalized_active.setup_receipt_sha256 = hash(setupBytes); + else report.normalized_active.observe_receipt_sha256 = hash(setupBytes); } } before = normalizedReadback(report.goal_path, fixture, normalizedExpected); @@ -1867,22 +1937,58 @@ export async function runProbe(op, config) { let active = true; let resolveOutcome; const outcome = new Promise(resolve => { resolveOutcome = resolve; }); + const boundedNative = entry?.native_opportunity_limit === 2; + if (boundedNative) { + report.native_opportunities = []; + completedNativeOpportunity = () => report.native_opportunities[0]?.kind + || (messageId === undefined + && buffered.filter(event => event.type === "user.message").length === 1 + && continuationCompletion(buffered, + buffered.find(event => event.type === "user.message")?.data?.messageId, true)?.kind); + } + const finishOutcome = terminal => { + if (boundedNative) nativeObservationClosed = true; + resolveOutcome(terminal); + }; + if (exactRecordedPrompt !== undefined) promptFailure = error => finishOutcome({ error }); if (continuation) pendingOutcome = () => { - if (messageId !== undefined) resolveOutcome({ pending: true }); + if (messageId !== undefined) finishOutcome({ pending: true }); }; const consume = event => { if (!active || event.agentId || event.data?.parentToolCallId) return; if (event.type === "session.error") { - resolveOutcome({ error: new Error(event.data?.message ?? "Native root session error") }); + finishOutcome({ error: new Error(event.data?.message ?? "Native root session error") }); } else if (event.type === "session.idle" && event.data?.aborted === true) { - resolveOutcome({ error: new Error("Native session idle was aborted") }); + finishOutcome({ error: new Error("Native session idle was aborted") }); } else if (messageId === undefined) { buffered.push(event); } else if (continuation) { completionEvents.push(event); - const terminal = event.type === "session.idle" - ? continuationCompletion(completionEvents, messageId) : undefined; - if (terminal) resolveOutcome(terminal); + if (boundedNative) { + const users = completionEvents.filter(item => item.type === "user.message"); + if (users.length > 2 || (users.length && users[0].data?.messageId !== messageId)) { + finishOutcome({ error: new Error("Unexpected native continuation root event") }); + return; + } + const index = report.native_opportunities.length; + const user = users[index]; + // Native autopilot retains the initiating send's origin across interactions. + const terminal = user && continuationCompletion( + completionEvents.slice(completionEvents.indexOf(user)), user.data?.messageId, true, messageId); + if (terminal?.error) finishOutcome(terminal); + else if (terminal) { + report.native_opportunities.push({ message_id: user.data.messageId, kind: terminal.kind, + message_event_id: terminal.message.id, turn_end_event_id: terminal.turn_end?.id }); + if (terminal.kind === "task_complete" || report.native_opportunities.length === 2) { + report.native_observation_stop = terminal.kind === "task_complete" ? "task_complete" : "opportunity_limit"; + finishOutcome(terminal); + } + } + } else { + const terminal = event.type === "session.idle" + ? continuationCompletion(completionEvents, messageId) : undefined; + if (terminal) resolveOutcome(terminal); + } } else if (event.type === "assistant.message") { matchingMessage = event.data?.originatingMessageId === messageId && typeof event.id === "string" && typeof event.data.content === "string" @@ -1902,12 +2008,14 @@ export async function runProbe(op, config) { const modelStarted = performance.now(); report.model_timing_source = "Owned SDK send through correlated native session.idle, not provider HTTP timing"; let deadlineTimer; + let abortedAtBoundary = false; try { try { const deadline = new Promise(resolve => { - deadlineTimer = setTimeout(() => resolve({ - error: new Error(`${report.timeout_ms}-millisecond SDK send/completion deadline exceeded`), - }), report.timeout_ms); + deadlineTimer = setTimeout(() => { + if (boundedNative) nativeObservationClosed = true; + resolve({ error: new Error(`${report.timeout_ms}-millisecond SDK send/completion deadline exceeded`) }); + }, report.timeout_ms); }); const sendRequest = report.recorded_autopilot_send ? { ...report.recorded_autopilot_send, mode: "immediate", wait: false } @@ -1921,14 +2029,14 @@ export async function runProbe(op, config) { report.model_message_id = id; if (!active) return; messageId = id; - if (continuation && report.pending_input?.length) resolveOutcome({ pending: true }); + if (continuation && report.pending_input?.length) finishOutcome({ pending: true }); for (const event of buffered) consume(event); buffered.length = 0; - }).catch(error => resolveOutcome({ error })); + }).catch(error => finishOutcome({ error })); const terminal = await Promise.race([outcome, deadline]); if (terminal.error) throw terminal.error; report.model_result = terminal.message; - report.model_completion_event = terminal.idle; + report.model_completion_event = terminal.idle ?? terminal.turn_end; if (continuation && !terminal.pending) { report.model_completion_kind = terminal.kind; if (terminal.task_complete) report.model_task_completion = terminal.task_complete; @@ -1936,13 +2044,19 @@ export async function runProbe(op, config) { if (terminal.pending) { report.pending_input_observed = true; await call("session.abort", undefined, () => session.abort()); + abortedAtBoundary = boundedNative; + } else if (boundedNative && terminal.kind !== "task_complete") { + await call("session.abort", undefined, () => session.abort()); + abortedAtBoundary = true; } } catch (error) { fail(error); throw error; } finally { active = false; + nativeObservationClosed = true; pendingOutcome = undefined; + promptFailure = undefined; clearTimeout(deadlineTimer); try { stopCompletion(); } catch (error) { fail(error); } report.model_completed_at = new Date().toISOString(); @@ -1963,9 +2077,17 @@ export async function runProbe(op, config) { throw new Error("Frozen continuation configuration changed"); } report.prefix_after = await readEvents(); + if (exactRecordedPrompt !== undefined + && (report.recorded_prompt_transform.invocations !== + (boundedNative ? 1 + report.recorded_prompt_transform.native_passthrough.length : 1) + || !report.recorded_prompt_transform.input_verified + || report.recorded_prompt_transform.replacement_sha256 !== hash(exactRecordedPrompt))) { + throw new Error("Recorded transformed prompt hook did not verify exactly one replacement"); + } const correspondence = continuationCorrespondence( report.prefix_before, report.prefix_after, candidateEvents.slice(report.model_event_start_index), - report.model_message_id, report.bound_raw_trigger, report.recorded_autopilot_send); + report.model_message_id, report.bound_raw_trigger, report.recorded_autopilot_send, exactRecordedPrompt, + boundedNative ? report.recorded_prompt_transform.native_passthrough : undefined); Object.assign(report.checks, correspondence); report.primary_system_messages = correspondence.primary_system_messages; delete report.checks.primary_system_messages; @@ -1977,8 +2099,13 @@ export async function runProbe(op, config) { && store.current.id === normalizedExpected.current.id && store.current.objective === normalizedExpected.current.objective && isDeepStrictEqual(report.final_state.goal?.state, objectiveState(store)); - if (!report.checks.normalized_native_completion) { - throw new Error("Normalized canary requires genuine task_complete and matching completed native/persisted objective"); + if (entry.purpose === "setup-canary" && (boundedNative + ? report.native_opportunities.length !== 2 + || (report.model_completion_kind === "task_complete" && !report.checks.normalized_native_completion) + : !report.checks.normalized_native_completion)) { + throw new Error(boundedNative + ? "Two-opportunity canary requires two completed native opportunities and consistent task-complete state" + : "Normalized canary requires genuine task_complete and matching completed native/persisted objective"); } } report.tool_effects = candidateEvents.slice(report.model_event_start_index) @@ -2005,7 +2132,9 @@ export async function runProbe(op, config) { report.status = report.pending_input_observed ? "HUMAN_INPUT_PENDING" : "CANDIDATE_OBSERVED"; } catch (error) { fail(error); - try { await call("session.abort", undefined, () => session.abort()); } catch (abortError) { fail(abortError); } + if (!abortedAtBoundary) { + try { await call("session.abort", undefined, () => session.abort()); } catch (abortError) { fail(abortError); } + } } }; try { @@ -2154,6 +2283,62 @@ export async function runProbe(op, config) { onElicitationRequest: request => recordPending("elicitation", request), }; delete candidateOptions.availableTools; + const fixture = contextFixture(config); + if (fixture.runtime_treatment === cappedPublicResume && fixture.autopilot_send) { + exactRecordedPrompt = fixture.canonical.chat_messages.at(-1).content; + report.recorded_prompt_transform = { + surface: "resumeOptions.hooks.onUserPromptTransformed", + source_event_id: fixture.source_binding.cutoff_event_id, + canonical_sha256: fixture.canonical_sha256, + expected_sha256: hash(exactRecordedPrompt), invocations: 0, + ...(entry.native_opportunity_limit === 2 ? { native_passthrough: [], held_after_boundary: 0 } : {}), + }; + candidateOptions.hooks = { + ...candidateOptions.hooks, + onUserPromptTransformed(input, invocation) { + const observed = report.recorded_prompt_transform; + if (entry.native_opportunity_limit === 2 + && (nativeObservationClosed || observed.invocations >= 2 + || completedNativeOpportunity?.() === "task_complete")) { + observed.held_after_boundary++; + return new Promise(() => {}); + } + observed.invocations++; + if (entry.native_opportunity_limit === 2 && observed.invocations === 2) { + if (completedNativeOpportunity?.() !== "assistant_final" || report.model_send_calls !== 1 + || input?.sessionId !== config.session_id || invocation?.sessionId !== config.session_id + || typeof input.prompt !== "string" + || typeof input.transformedPrompt !== "string" || !input.transformedPrompt) { + const error = new Error("Native continuation requires a completed first native opportunity"); + fail(error); + promptFailure?.(error); + return new Promise(() => {}); + } + observed.native_passthrough.push({ + prompt_sha256: hash(input.prompt), transformed_sha256: hash(input.transformedPrompt), + }); + return {}; + } + const wrapper = typeof input?.transformedPrompt === "string" + ? input.transformedPrompt.match(/([0-9T:.+Z-]+)<\/current_datetime>\n\n/) : null; + if (observed.invocations !== 1 || report.model_send_calls !== 1 + || input?.sessionId !== config.session_id || invocation?.sessionId !== config.session_id + || input?.prompt !== fixture.autopilot_send.prompt + || !wrapper || !Number.isFinite(Date.parse(wrapper[1])) + || !input.transformedPrompt.includes(fixture.autopilot_send.prompt)) { + const error = new Error("Recorded transformed prompt hook input differs from the once-only source-bound send"); + fail(error); + promptFailure?.(error); + // The pinned SDK swallows throws. Hold this hook while the existing abort/shutdown path runs. + return new Promise(() => {}); + } + observed.input_verified = true; + observed.input_sha256 = hash(input.transformedPrompt); + observed.replacement_sha256 = hash(exactRecordedPrompt); + return { modifiedTransformedPrompt: exactRecordedPrompt }; + }, + }; + } } report.connection.mode = "copilot-cli"; report.connection.workingDirectory = config.cwd; diff --git a/skills/skill-evaluation/scripts/native_workflow_probe.test.mjs b/skills/skill-evaluation/scripts/native_workflow_probe.test.mjs index 8e3d215..e063d53 100644 --- a/skills/skill-evaluation/scripts/native_workflow_probe.test.mjs +++ b/skills/skill-evaluation/scripts/native_workflow_probe.test.mjs @@ -72,6 +72,11 @@ export class CopilotClient { let goalStore; try { goalStore = JSON.parse(readFileSync(goalPath, "utf8")); current = goalStore.current; } catch {} const persistGoal = () => writeFileSync(goalPath, JSON.stringify(goalStore)); + state.completeGoal = () => { + current.status = "completed"; + current.completionSummary = "Synthetic completion"; + persistGoal(); + }; if (current && state.pauseOnCold && current.status === "active") { Object.assign(current, { status: "paused", pauseReason: "resumed_session", updatedAt: new Date().toISOString() }); @@ -96,22 +101,36 @@ export class CopilotClient { state.sentId = id; state.sendReady?.(); if (state.sendFails) throw new Error("Test send transport failure"); - if (state.sendHook) { - const messageId = state.sendHook(id, emit); - if (state.completeOnSend) { - current.status = "completed"; - current.completionSummary = "Synthetic completion"; - persistGoal(); + const finish = () => { + if (state.sendHook) { + const messageId = state.sendHook(id, emit); + if (state.completeOnSend) { + current.status = "completed"; + current.completionSummary = "Synthetic completion"; + persistGoal(); + } + return messageId; } - return messageId; + emit("assistant.turn_start", { model: options.model }); + emit("assistant.usage", { apiCallId: "test-api-1", model: options.model }); + emit("assistant.usage", { apiCallId: "test-api-2", model: options.model }); + emit("assistant.message", { content: "Test SDK response", originatingMessageId: id }); + emit("assistant.idle", {}); + emit("session.idle", { mode: "autopilot" }); + return id; + }; + if (options.hooks?.onUserPromptTransformed) { + void Promise.resolve().then(async () => { + const input = { sessionId: session.sessionId, + prompt: message.prompt, + transformedPrompt: "2026-02-02T00:00:00.000Z\\n\\n" + message.prompt }; + const output = await options.hooks.onUserPromptTransformed(input, { sessionId: session.sessionId }); + state.transformedPrompt = output?.modifiedTransformedPrompt ?? input.transformedPrompt; + finish(); + }).catch(error => emit("session.error", { message: error.message })); + return id; } - emit("assistant.turn_start", { model: options.model }); - emit("assistant.usage", { apiCallId: "test-api-1", model: options.model }); - emit("assistant.usage", { apiCallId: "test-api-2", model: options.model }); - emit("assistant.message", { content: "Test SDK response", originatingMessageId: id }); - emit("assistant.idle", {}); - emit("session.idle", { mode: "autopilot" }); - return id; + return finish(); }, rpc: { commands: { invoke: async request => { @@ -864,7 +883,8 @@ async function continuationFixture(t, { loadedCopies = false, operational = fals const persist = (type, data) => state.events.push(emit(type, data)); persist("user.message", { ...(autopilot ? { content: "", source: "autopilot", agentMode: "autopilot", - transformedContent: `2026-02-02T00:00:00.000Z\n\n${owned.request.prompt}`, + transformedContent: state.transformedPrompt + ?? `2026-02-02T00:00:00.000Z\n\n${owned.request.prompt}`, } : { content: "Synthetic trigger" }), messageId: id, interactionId: id }); emit("system.message", { interactionId: id, content: input.cli_preamble @@ -1293,6 +1313,168 @@ async function normalizedCanaryFixture(t) { return { ...owned, setupConfig }; } +async function twoOpportunityFixture(t) { + const owned = await normalizedCanaryFixture(t); + const { config, store, source, goal, locator, input, save, state } = owned; + Object.assign(store.current, { continuationCount: 10, turnCount: 837, tokenCount: 128149341, + creditCountNanoAiu: "3137066880000", resumeCount: 0 }); + delete store.current.creditLimit; + const bytes = JSON.stringify(store); + writeFileSync(source, bytes); + writeFileSync(goal, bytes); + locator.active_objective.sha256 = createHash("sha256").update(bytes).digest("hex"); + locator.runtime_treatment = "capped-public-resume-v1"; + locator.provenance = "synthetic-continuation-canary"; + writeFileSync(config.context_fixture_file, JSON.stringify(locator)); + input.native_opportunity_limit = 2; + save(); + state.completeViaTool = false; + state.completeOnSend = false; + assert.equal((await runProbe("setup", owned.setupConfig)).status, "SETUP_OBSERVED"); + return owned; +} + +test("bounded native observation binds its selector and rejects invalid or uncapped choices before send", async t => { + const { config, root, input, save, state, locator } = await twoOpportunityFixture(t); + const selected = continuationEntry(config); + delete input.native_opportunity_limit; + save(); + const original = continuationEntry(config); + assert.notEqual(selected.configuration_sha256, original.configuration_sha256); + assert.equal(selected.binding.native_opportunity_limit, 2); + assert.equal(original.binding.native_opportunity_limit, undefined); + for (const value of [true, 1, 3, 2.5, "2", null]) { + config.output = join(root, `invalid-${randomUUID()}.json`); + input.native_opportunity_limit = value; + save(); + const result = await runProbe("candidate", config); + assert.equal(result.status, "PREMODEL_BLOCKED"); + assert.match(result.error.message, /native_opportunity_limit/); + } + input.native_opportunity_limit = 2; + locator.runtime_treatment = "public-command-normalized-active-v1"; + delete locator.provenance; + writeFileSync(config.context_fixture_file, JSON.stringify(locator)); + save(); + assert.throws(() => continuationEntry(config), /requires capped/); + assert.equal(state.sendCalls, 0); +}); + +for (const scenario of ["second-final", "empty-native-raw", "first-task", "second-task", "protected-input", "native-error", + "aborted", "deadline", "early-prompt", "substituted-root", "substituted-origin", "unmatched-turn", + "missing-end-idle", "unmatched-end-idle", "nonfinal-idle"]) { + test(`bounded native observation owning capture: ${scenario}`, async t => { + const { config, input, save, state, goal } = await twoOpportunityFixture(t); + if (["deadline", "substituted-origin", "unmatched-turn"].includes(scenario)) { input.timeout_ms = 25; save(); } + if (scenario === "first-task") { + state.completeViaTool = true; + state.completeOnSend = true; + } + const first = state.sendHook; + state.forwardedNativePrompts = 0; + state.thirdModels = 0; + state.sendHook = (id, emit) => { + const persist = (type, data) => state.events.push(emit(type, data)); + first(id, (type, data, extra) => { + if (type === "session.idle") return { id: randomUUID(), type, data }; + if (type === "assistant.message" && scenario !== "first-task") { + Object.assign(data, { turnId: "first", phase: "final_answer", toolRequests: [] }); + } + return emit(type, data, extra); + }); + state.events = state.events.filter(event => event.type !== "session.idle"); + if (scenario === "deadline" || scenario === "first-task") return id; + if (scenario !== "early-prompt") persist("assistant.turn_end", { turnId: "first" }); + const hook = state.resumes.at(-1).hooks.onUserPromptTransformed; + const next = { sessionId: config.session_id, prompt: "Synthetic native continuation", + transformedPrompt: "2026-02-02T00:00:01.000Z\n\nSynthetic native continuation" }; + if (scenario === "empty-native-raw") next.prompt = ""; + void Promise.resolve(hook(next, { sessionId: config.session_id })).then(output => { + state.forwardedNativePrompts++; + assert.equal(output?.modifiedTransformedPrompt, undefined); + const second = randomUUID(); + persist("user.message", { content: "", source: "autopilot", agentMode: "autopilot", + transformedContent: scenario === "substituted-root" ? "Substituted native input" : next.transformedPrompt, + messageId: second, interactionId: second }); + emit("system.message", { interactionId: second, content: input.cli_preamble + + "\nYou are an interactive tool that helps users with software engineering tasks." }); + if (scenario === "protected-input") { + void state.resumes.at(-1).onUserInputRequest({ question: "Synthetic protected input" }); + } else if (scenario === "native-error") { + emit("session.error", { message: "Synthetic native continuation failure" }); + } else if (scenario === "aborted") { + emit("session.idle", { aborted: true }); + } else if (scenario === "second-task") { + const request = { name: "task_complete", toolCallId: "second-task", arguments: { summary: "Synthetic completion" } }; + persist("assistant.message", { content: null, originatingMessageId: id, interactionId: second, + turnId: "second", toolRequests: [request] }); + persist("tool.execution_start", { toolName: request.name, toolCallId: request.toolCallId, + arguments: request.arguments, turnId: "second" }); + state.completeGoal(); + persist("tool.execution_complete", { toolCallId: request.toolCallId, success: true, + interactionId: second, turnId: "second", result: { content: "Synthetic completion" } }); + persist("assistant.turn_end", { turnId: "second" }); + } else { + persist("assistant.message", { content: "Synthetic second completed opportunity.", + originatingMessageId: scenario === "substituted-origin" ? randomUUID() : id, + interactionId: second, turnId: "second", + phase: scenario === "nonfinal-idle" ? "commentary" : "final_answer", toolRequests: [] }); + if (scenario !== "missing-end-idle") { + persist("assistant.turn_end", { turnId: ["unmatched-turn", "unmatched-end-idle"].includes(scenario) + ? "unrelated" : "second" }); + } + if (scenario.endsWith("-idle")) persist("session.idle", {}); + void Promise.resolve(hook(next, { sessionId: config.session_id })).then(() => { state.thirdModels++; }); + } + }); + return id; + }; + const result = await runProbe("candidate", config); + const expected = ["second-final", "empty-native-raw", "second-task"].includes(scenario) ? "CANDIDATE_OBSERVED" + : scenario === "protected-input" ? "HUMAN_INPUT_PENDING" : "FAILED"; + assert.equal(result.status, expected, result.error?.message ?? result.status); + assert.equal(state.sendCalls, 1); + assert.equal(state.thirdModels, 0); + assert.equal(JSON.parse(readFileSync(goal)).current.status, + ["first-task", "second-task"].includes(scenario) ? "completed" : "active"); + assert.equal(state.aborts, scenario === "second-task" ? 0 : 1); + if (["second-final", "empty-native-raw"].includes(scenario)) { + assert.equal(result.native_opportunities.length, 2); + assert.equal(result.model_result.data.originatingMessageId, result.model_message_id); + assert.equal(result.native_observation_stop, "opportunity_limit"); + assert.equal(result.model_completion_event.type, "assistant.turn_end"); + assert.equal(result.checks.exactly_once, true); + assert.equal(result.checks.native_continuation_passthrough, true); + assert.equal(result.checks.normalized_native_completion, false); + assert.equal(result.recorded_prompt_transform.held_after_boundary, 1); + assert.equal(result.continuation_receipt.status, "CONTINUATION_CANARY_OBSERVED"); + } else if (scenario === "early-prompt") { + assert.equal(state.forwardedNativePrompts, 0); + assert.match(result.error.message, /completed first native opportunity/); + } else if (scenario === "substituted-root") { + assert.match(result.error.message, /native continuation/); + } else if (["deadline", "substituted-origin", "unmatched-turn"].includes(scenario)) { + assert.match(result.error.message, /deadline/); + } else if (scenario.endsWith("-idle")) { + assert.equal(result.native_opportunities.length, 1); + assert.match(result.error.message, /matching turn end/); + } else if (scenario === "native-error") { + assert.match(result.error.message, /Synthetic native continuation failure/); + } else if (scenario === "aborted") { + assert.match(result.error.message, /Native session idle was aborted/); + } else if (scenario === "first-task") { + assert.equal(state.forwardedNativePrompts, 0); + assert.equal(result.native_opportunities.length, 1); + assert.equal(result.model_completion_kind, "task_complete"); + assert.match(result.error.message, /Two-opportunity canary/); + } else if (scenario === "second-task") { + assert.equal(result.native_opportunities.length, 2); + assert.equal(result.model_completion_kind, "task_complete"); + assert.equal(result.checks.normalized_native_completion, true); + } + }); +} + test("normalized canary witnesses both cold pauses and resumes without sending the command prompt", async t => { const { config, setupConfig, state, store, goal, request, input, save } = await normalizedCanaryFixture(t); const setup = await runProbe("setup", setupConfig); diff --git a/skills/skill-evaluation/scripts/session_restore.py b/skills/skill-evaluation/scripts/session_restore.py index 74ef3d7..f318d8b 100644 --- a/skills/skill-evaluation/scripts/session_restore.py +++ b/skills/skill-evaluation/scripts/session_restore.py @@ -37,6 +37,7 @@ def active_objective_state(store: dict) -> dict: NORMALIZED_ACTIVE_CANARY = "public-command-normalized-active-v1" +CAPPED_PUBLIC_RESUME = "capped-public-resume-v1" def normalized_active_canary(store: dict, source: str) -> None: @@ -48,6 +49,18 @@ def normalized_active_canary(store: dict, source: str) -> None: raise ValueError("Normalized active treatment requires the capped synthetic canary") +def capped_public_resume(store: dict, source: str, provenance: str) -> None: + active_objective_state(store) + synthetic = provenance == "synthetic-continuation-canary" + if (provenance not in ("synthetic-continuation-canary", "source-bound-adapted-public-resume") + or not isinstance(source, str) or not source.startswith("/") + or not source.endswith("/events.jsonl") + or (synthetic and not re.fullmatch(r"/synthetic/[0-9a-f-]{36}/events\.jsonl", source)) + or (not synthetic and source.startswith("/synthetic/")) + or "creditLimit" in store["current"]): + raise ValueError("Adapted capped public resume requires an initially uncapped active objective and matching provenance") + + def _objective_state(store: dict, status: str) -> dict: required = {"id", "objective", "status", "autopilotOrigin", "continuationCount", "turnCount", "tokenCount", "creditCountNanoAiu", "resumeCount", "createdAt", "updatedAt"} diff --git a/skills/skill-evaluation/scripts/skill_eval.py b/skills/skill-evaluation/scripts/skill_eval.py index abf420e..9b9d57e 100644 --- a/skills/skill-evaluation/scripts/skill_eval.py +++ b/skills/skill-evaluation/scripts/skill_eval.py @@ -2494,6 +2494,9 @@ def append(kind, data): } if "runtime_treatment" in historical: locator["runtime_treatment"] = historical["runtime_treatment"] + from session_restore import CAPPED_PUBLIC_RESUME + if historical["runtime_treatment"] == CAPPED_PUBLIC_RESUME: + locator["provenance"] = historical["provenance"] if objective_key in historical: locator[objective_key] = { "path": str(owner / f"{objective_key.replace('_', '-')}.json"), @@ -2834,23 +2837,25 @@ def _validate_recorded_context_fixture(fixture: dict, root: Path | None) -> dict from session_restore import ( active_objective_state, compile_task_snapshot, completed_objective_state, permission_rules, recorded_autopilot_request, recorded_native_payloads, - NORMALIZED_ACTIVE_CANARY, normalized_active_canary, + NORMALIZED_ACTIVE_CANARY, normalized_active_canary, CAPPED_PUBLIC_RESUME, capped_public_resume, ) keys = {"schema_version", "provenance", "recorded_context", "restoration", "trigger", "consumer", "snapshot", "required_tools", "limitations", "personal_instructions", "candidate_permissions"} autopilot = "recorded_autopilot_request" in fixture normalized = "runtime_treatment" in fixture + capped = fixture.get("runtime_treatment") == CAPPED_PUBLIC_RESUME + provenances = (("synthetic-continuation-canary", "source-bound-adapted-public-resume") if capped else + ("synthetic-continuation-canary",) if normalized else ("source-bound-historical",)) if (set(fixture) - {"completed_objective", "active_objective", "recorded_autopilot_request", "runtime_treatment"} != keys or ("active_objective" in fixture) != autopilot or (autopilot and "completed_objective" in fixture) or (normalized and (not autopilot - or fixture["runtime_treatment"] != NORMALIZED_ACTIVE_CANARY)) + or fixture["runtime_treatment"] not in (NORMALIZED_ACTIVE_CANARY, CAPPED_PUBLIC_RESUME))) or type(fixture["schema_version"]) is not int or fixture["schema_version"] != 3 - or fixture["provenance"] != ( - "synthetic-continuation-canary" if normalized else "source-bound-historical") + or fixture["provenance"] not in provenances or root is None or not isinstance(fixture["personal_instructions"], str) or not fixture["personal_instructions"].strip() @@ -2959,7 +2964,10 @@ def _validate_recorded_context_fixture(fixture: dict, root: Path | None) -> dict read_owner_json(request_path), event, trigger, store) objective["recorded_autopilot_request_bytes"] = request_bytes if normalized: - normalized_active_canary(store, context["source"]) + if capped: + capped_public_resume(store, context["source"], fixture["provenance"]) + else: + normalized_active_canary(store, context["source"]) permission_rules(fixture["candidate_permissions"]) return {**fixture, "snapshot": compile_task_snapshot(fixture["snapshot"]), "canonical": context, "canonical_bytes": source_bytes, @@ -3118,13 +3126,21 @@ def _run_native_context_calibration( raise ValueError("Context fixture requires an owner-only unaliased single-link file") fixture = read_owner_json(fixture_file) normalized = isinstance(fixture, dict) and "runtime_treatment" in fixture + from session_restore import CAPPED_PUBLIC_RESUME + capped = isinstance(fixture, dict) and fixture.get("runtime_treatment") == CAPPED_PUBLIC_RESUME historical = isinstance(fixture, dict) and ( fixture.get("provenance") == "source-bound-historical" or normalized) if normalized: - if continuation_file is None or source_preparation_only or not model_send: + zero_model_capped_preparation = ( + capped and fixture.get("provenance") == "source-bound-adapted-public-resume" + and source_preparation_only and continuation_file is None and not model_send) + if (not zero_model_capped_preparation + and (continuation_file is None or source_preparation_only or not model_send)): raise ValueError("Normalized active treatment requires the explicit continuation canary") result["runtime_treatment"] = fixture["runtime_treatment"] result["historical_active_equivalence"] = False + if capped: + result["provenance"] = fixture["provenance"] calibration_session_id = None if source_preparation_only: if not historical or model_send: @@ -3138,7 +3154,9 @@ def _run_native_context_calibration( or any(path.is_symlink() for path in (personal_skill_dir, *personal_skill_dir.parents))): raise ValueError("Personal skill discovery requires an unaliased historical definition directory") if historical: - result["claim"] = ("Synthetic normalized-active infrastructure only; historical equivalence unproved" + result["claim"] = ("Adapted source-bound capped public-resume setup only; historical active equivalence false" + if capped and fixture["provenance"] == "source-bound-adapted-public-resume" else + "Synthetic normalized-active infrastructure only; historical equivalence unproved" if normalized else "Historical setup observations only; behavioral admission and skill improvement unproved") if model_send and continuation_file is None: @@ -3372,6 +3390,12 @@ def execute_import(command): result["claim"] = ("Synthetic continuation capability observation; historical reproduction and skill efficacy unproved" if result.get("candidate", {}).get("continuation_entry", {}).get("purpose") == "setup-canary" else "Historical continuation observation; complete baseline fidelity and skill efficacy unproved") + if result.get("runtime_treatment") == "capped-public-resume-v1": + result["claim"] = ( + "Adapted source-bound capped public-resume observation; historical active equivalence false; " + "behavioral admission and skill efficacy unproved" + if result.get("provenance") == "source-bound-adapted-public-resume" else + "Synthetic capped public-resume capability observation only; historical behavior unproved") write_json(run_root / "context-result.json", result) return result