From ee32568938e86189e2f91f25f7345dea87646b05 Mon Sep 17 00:00:00 2001 From: "@daniel-lxs" <57051444+daniel-lxs@users.noreply.github.com> Date: Sat, 5 Sep 2026 19:01:24 +0000 Subject: [PATCH 1/2] feat: offer opt-in required-check follow-up in Fast --- .../__tests__/fast-agent-prompt.test.ts | 103 ++++++++++++++++++ .../server/fast-agent/fast-agent-prompt.ts | 5 + 2 files changed, 108 insertions(+) diff --git a/packages/cloud-agents/src/server/fast-agent/__tests__/fast-agent-prompt.test.ts b/packages/cloud-agents/src/server/fast-agent/__tests__/fast-agent-prompt.test.ts index da230777a..df2bca8c0 100644 --- a/packages/cloud-agents/src/server/fast-agent/__tests__/fast-agent-prompt.test.ts +++ b/packages/cloud-agents/src/server/fast-agent/__tests__/fast-agent-prompt.test.ts @@ -358,6 +358,109 @@ describe('buildFastAgentSystemPrompt', () => { ); }); + it('offers pending PR check follow-up once without scheduling before opt-in', () => { + const prompt = buildFastAgentSystemPrompt({ availableEnvironments: [] }); + + expect(prompt).toContain( + 'reliable current evidence shows CI is still pending on an open PR', + ); + expect(prompt).toContain( + 'Want me to let you know when the required checks pass?', + ); + expect(prompt).toContain('Do not offer again after a decline'); + expect(prompt).toContain( + 'create no wakeup until the user explicitly accepts or requests this monitoring', + ); + expect(prompt).toContain('do not inspect or schedule from them'); + }); + + it('requires authoritative required-check capability rather than inferring green CI', () => { + const prompt = buildFastAgentSystemPrompt({ availableEnvironments: [] }); + + expect(prompt).toContain( + 'inspect the available integration tool schemas and results', + ); + expect(prompt).toContain('not CI results or required-check policy'); + expect(prompt).toContain( + 'never invent a tool or infer requirements from observed checks', + ); + expect(prompt).toContain( + 'do not schedule or promise required-check success monitoring', + ); + expect(prompt).toContain('nonempty authoritative required-check set'); + expect(prompt).toContain( + 'Empty, unknown, missing, partial, stale or pending check data is not success', + ); + expect(prompt).toContain( + 'neither mergeability nor all observed checks passing proves required checks passed', + ); + }); + + it('pins opt-in monitors to an exact revision with a quiet bounded schedule', () => { + const prompt = buildFastAgentSystemPrompt({ availableEnvironments: [] }); + + expect(prompt).toContain('schedule "every 10m x12" (at most two hours)'); + expect(prompt).toContain('reportPolicy "only_when_notable"'); + expect(prompt).toContain( + 'same source-control provider, repository, PR and head SHA even if its wording differs', + ); + expect(prompt).toContain( + 'the required-check evidence source, stop conditions and reporting rules in the stored wakeup prompt', + ); + expect(prompt).toContain('never silently follow a newer revision'); + }); + + it('keeps stop conditions and precise success wording in wakeup continuation turns', () => { + const prompt = buildFastAgentSystemPrompt({ + availableEnvironments: [], + turnSource: 'platform_event', + platformEventKind: 'scheduled_wakeup', + }); + + expect(prompt).toContain('must first re-read PR state and head SHA'); + expect(prompt).toContain( + 'Cancel on a merged or closed PR, a superseded head SHA, or user cancellation', + ); + expect(prompt).toContain( + 'If the capability becomes unavailable, cancel and explain the blocker once', + ); + expect(prompt).toContain( + 'Only after revalidating that the PR is still open on the pinned SHA', + ); + expect(prompt).toContain( + 'Required checks passed for # at .', + ); + expect(prompt).toContain( + 'Never say "ready to merge", imply review approval, or merge from this monitor', + ); + expect(prompt).toContain('Cancel after success'); + expect(prompt).toContain( + 'on the final run, stop without renewal and report an unresolved timeout once', + ); + expect(prompt).toContain('a finished wakeup needs no cancel'); + }); + + it('preserves existing event-driven follow-through instead of duplicating notifications', () => { + const prompt = buildFastAgentSystemPrompt({ availableEnvironments: [] }); + + expect(prompt).toContain( + 'List active wakeups first and reuse an equivalent monitor', + ); + expect(prompt).toContain('stay quiet on unchanged results'); + expect(prompt).toContain( + 'leave failed-CI reporting and remediation to existing logic', + ); + expect(prompt).toContain( + 'Do not duplicate task completion, PR review notifications or their retries, merged/closed PR tracking, or failed-CI triage with wakeups', + ); + expect(prompt).toContain( + 'cancel it when tools are permitted (otherwise at its next wakeup)', + ); + expect(prompt).toContain( + 'without repeating an already-delivered notification', + ); + }); + it('lists on-demand servers by name with their tool names instead of mounting them', () => { const prompt = buildFastAgentSystemPrompt({ availableEnvironments: [], diff --git a/packages/cloud-agents/src/server/fast-agent/fast-agent-prompt.ts b/packages/cloud-agents/src/server/fast-agent/fast-agent-prompt.ts index 19fc00595..8f73856b7 100644 --- a/packages/cloud-agents/src/server/fast-agent/fast-agent-prompt.ts +++ b/packages/cloud-agents/src/server/fast-agent/fast-agent-prompt.ts @@ -351,6 +351,11 @@ ${reactionGuidance} - Call a deployment MCP tool when it can answer the request. Fast receives the same actor-authorized remote and deployment-proxied MCP tool catalog as delegated tasks; local stdio servers remain sandbox-only. Servers listed with a tool prefix expose each tool individually with its native JSON schema. On-demand servers are reached through \`find_integration_tools\` (fetch the schema by server id and tool name, or search by keywords) followed by \`call_integration_tool\`; the same acknowledgement, duplicate, and audit rules apply to both paths. - Use \`roomote_manage_custom_automations\` for custom automation lifecycle requests. It uses the current user's deployment authorization, is admin-only, and is unavailable to advisor and judge subagents. List before modifying an existing automation, use "list_models" before setting a model override, use update with "enabled" to enable or disable, and use "run_now" rather than "launch_task" to test an automation. Communicate first on a human-authored turn; platform events remain exempt. Delete only when the user explicitly requests it, and after creating an automation ask whether they want to run it now. - Use "manage_wakeups" when the user wants a reminder, a delayed follow-up, or a recurring check that reports back into this conversation ("remind me in 20 minutes", "check every 10 minutes until CI is green", "every weekday at 9 ping me with open PRs"). The schedule is one short string: "in s|m|h|d" for a reminder, "every s|m|h|d" for a repeating check, "cron 0 9 * * 1-5" for a five-field calendar schedule. Prefer "in 30s", not fractional "in 0.5m". Recurring intervals under five minutes require an x or until bound, such as "every 30s x3". Delivery is best effort; never promise an exact 30-second reply. Send only the fields the action needs. It is scoped to this conversation and available to every participant. Do not use \`roomote_manage_custom_automations\` for conversation-scoped reminders, and never sleep or poll inside a turn instead of scheduling a wakeup. After creating one, confirm the plan and the next run time in one sentence; when the user says stop or cancel, use action "cancel". +- When PR work finishes and reliable current evidence shows CI is still pending on an open PR, offer once in the delivery closeout: "Want me to let you know when the required checks pass?" Do not offer again after a decline, on unchanged updates, or when an equivalent monitor already exists. This is an opt-in offer, not permission to poll: create no wakeup until the user explicitly accepts or requests this monitoring. Presentation-only events still only present their supplied information; do not inspect or schedule from them. +- Before scheduling required-check monitoring, inspect the available integration tool schemas and results. Roomote's "get_pull_request" supplies PR state and head SHA, not CI results or required-check policy. Require an available source to identify the authoritative required checks for the PR's target and return complete check results for its exact head SHA; never invent a tool or infer requirements from observed checks. If required-check determination is unavailable, explain that limitation and do not schedule or promise required-check success monitoring. +- On explicit opt-in with that capability, use conversation-scoped "manage_wakeups", schedule "every 10m x12" (at most two hours), and reportPolicy "only_when_notable". List active wakeups first and reuse an equivalent monitor for the same source-control provider, repository, PR and head SHA even if its wording differs. Pin those exact identifiers, the required-check evidence source, stop conditions and reporting rules in the stored wakeup prompt; never silently follow a newer revision. Confirm the bound as well as the next run time. +- Each required-check wakeup must first re-read PR state and head SHA. Cancel on a merged or closed PR, a superseded head SHA, or user cancellation; never report check success for an obsolete revision. Otherwise verify a nonempty authoritative required-check set and complete, current results for every required check on the pinned SHA. Empty, unknown, missing, partial, stale or pending check data is not success; neither mergeability nor all observed checks passing proves required checks passed. If the capability becomes unavailable, cancel and explain the blocker once. Only after revalidating that the PR is still open on the pinned SHA, report once: "Required checks passed for # at ." Never say "ready to merge", imply review approval, or merge from this monitor. Cancel after success; on the final run, stop without renewal and report an unresolved timeout once rather than claiming success. +- Required-check monitors stay quiet on unchanged results and leave failed-CI reporting and remediation to existing logic. Do not duplicate task completion, PR review notifications or their retries, merged/closed PR tracking, or failed-CI triage with wakeups. Existing events remain authoritative; if they make a monitor obsolete, cancel it when tools are permitted (otherwise at its next wakeup), without repeating an already-delivered notification. ${recurringAutomationGuidance} - You may make multiple deployment MCP calls when needed, one at a time. Stop as soon as you have enough evidence and never repeat an identical call. From cf0cb045765cd767ddd05b6bb64f82f1ca37eff0 Mon Sep 17 00:00:00 2001 From: "@daniel-lxs" <57051444+daniel-lxs@users.noreply.github.com> Date: Sat, 5 Sep 2026 19:09:17 +0000 Subject: [PATCH 2/2] fix: verify required-check capability before offering monitoring --- .../__tests__/fast-agent-prompt.test.ts | 17 +++++++++++++++++ .../src/server/fast-agent/fast-agent-prompt.ts | 4 ++-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/packages/cloud-agents/src/server/fast-agent/__tests__/fast-agent-prompt.test.ts b/packages/cloud-agents/src/server/fast-agent/__tests__/fast-agent-prompt.test.ts index df2bca8c0..b43709c15 100644 --- a/packages/cloud-agents/src/server/fast-agent/__tests__/fast-agent-prompt.test.ts +++ b/packages/cloud-agents/src/server/fast-agent/__tests__/fast-agent-prompt.test.ts @@ -368,6 +368,9 @@ describe('buildFastAgentSystemPrompt', () => { 'Want me to let you know when the required checks pass?', ); expect(prompt).toContain('Do not offer again after a decline'); + expect(prompt).toContain( + 'the required-check capability above has been verified', + ); expect(prompt).toContain( 'create no wakeup until the user explicitly accepts or requests this monitoring', ); @@ -381,6 +384,20 @@ describe('buildFastAgentSystemPrompt', () => { 'inspect the available integration tool schemas and results', ); expect(prompt).toContain('not CI results or required-check policy'); + expect(prompt).toContain( + 'Before offering or scheduling required-check monitoring', + ); + expect(prompt).toContain( + 'If required-check determination is unavailable, suppress the offer', + ); + expect(prompt).toContain( + 'Revalidate this capability on acceptance before scheduling', + ); + expect( + prompt.indexOf('Before offering or scheduling required-check monitoring'), + ).toBeLessThan( + prompt.indexOf('Want me to let you know when the required checks pass?'), + ); expect(prompt).toContain( 'never invent a tool or infer requirements from observed checks', ); diff --git a/packages/cloud-agents/src/server/fast-agent/fast-agent-prompt.ts b/packages/cloud-agents/src/server/fast-agent/fast-agent-prompt.ts index 8f73856b7..4c921c636 100644 --- a/packages/cloud-agents/src/server/fast-agent/fast-agent-prompt.ts +++ b/packages/cloud-agents/src/server/fast-agent/fast-agent-prompt.ts @@ -351,8 +351,8 @@ ${reactionGuidance} - Call a deployment MCP tool when it can answer the request. Fast receives the same actor-authorized remote and deployment-proxied MCP tool catalog as delegated tasks; local stdio servers remain sandbox-only. Servers listed with a tool prefix expose each tool individually with its native JSON schema. On-demand servers are reached through \`find_integration_tools\` (fetch the schema by server id and tool name, or search by keywords) followed by \`call_integration_tool\`; the same acknowledgement, duplicate, and audit rules apply to both paths. - Use \`roomote_manage_custom_automations\` for custom automation lifecycle requests. It uses the current user's deployment authorization, is admin-only, and is unavailable to advisor and judge subagents. List before modifying an existing automation, use "list_models" before setting a model override, use update with "enabled" to enable or disable, and use "run_now" rather than "launch_task" to test an automation. Communicate first on a human-authored turn; platform events remain exempt. Delete only when the user explicitly requests it, and after creating an automation ask whether they want to run it now. - Use "manage_wakeups" when the user wants a reminder, a delayed follow-up, or a recurring check that reports back into this conversation ("remind me in 20 minutes", "check every 10 minutes until CI is green", "every weekday at 9 ping me with open PRs"). The schedule is one short string: "in s|m|h|d" for a reminder, "every s|m|h|d" for a repeating check, "cron 0 9 * * 1-5" for a five-field calendar schedule. Prefer "in 30s", not fractional "in 0.5m". Recurring intervals under five minutes require an x or until bound, such as "every 30s x3". Delivery is best effort; never promise an exact 30-second reply. Send only the fields the action needs. It is scoped to this conversation and available to every participant. Do not use \`roomote_manage_custom_automations\` for conversation-scoped reminders, and never sleep or poll inside a turn instead of scheduling a wakeup. After creating one, confirm the plan and the next run time in one sentence; when the user says stop or cancel, use action "cancel". -- When PR work finishes and reliable current evidence shows CI is still pending on an open PR, offer once in the delivery closeout: "Want me to let you know when the required checks pass?" Do not offer again after a decline, on unchanged updates, or when an equivalent monitor already exists. This is an opt-in offer, not permission to poll: create no wakeup until the user explicitly accepts or requests this monitoring. Presentation-only events still only present their supplied information; do not inspect or schedule from them. -- Before scheduling required-check monitoring, inspect the available integration tool schemas and results. Roomote's "get_pull_request" supplies PR state and head SHA, not CI results or required-check policy. Require an available source to identify the authoritative required checks for the PR's target and return complete check results for its exact head SHA; never invent a tool or infer requirements from observed checks. If required-check determination is unavailable, explain that limitation and do not schedule or promise required-check success monitoring. +- Before offering or scheduling required-check monitoring, inspect the available integration tool schemas and results. Roomote's "get_pull_request" supplies PR state and head SHA, not CI results or required-check policy. Require an available source to identify the authoritative required checks for the PR's target and return complete check results for its exact head SHA; never invent a tool or infer requirements from observed checks. If required-check determination is unavailable, suppress the offer; if the user explicitly requests monitoring, explain that limitation and do not schedule or promise required-check success monitoring. Revalidate this capability on acceptance before scheduling. +- When PR work finishes and reliable current evidence shows CI is still pending on an open PR, and the required-check capability above has been verified, offer once in the delivery closeout: "Want me to let you know when the required checks pass?" Do not offer again after a decline, on unchanged updates, or when an equivalent monitor already exists. This is an opt-in offer, not permission to poll: create no wakeup until the user explicitly accepts or requests this monitoring. Presentation-only events still only present their supplied information; do not inspect or schedule from them. - On explicit opt-in with that capability, use conversation-scoped "manage_wakeups", schedule "every 10m x12" (at most two hours), and reportPolicy "only_when_notable". List active wakeups first and reuse an equivalent monitor for the same source-control provider, repository, PR and head SHA even if its wording differs. Pin those exact identifiers, the required-check evidence source, stop conditions and reporting rules in the stored wakeup prompt; never silently follow a newer revision. Confirm the bound as well as the next run time. - Each required-check wakeup must first re-read PR state and head SHA. Cancel on a merged or closed PR, a superseded head SHA, or user cancellation; never report check success for an obsolete revision. Otherwise verify a nonempty authoritative required-check set and complete, current results for every required check on the pinned SHA. Empty, unknown, missing, partial, stale or pending check data is not success; neither mergeability nor all observed checks passing proves required checks passed. If the capability becomes unavailable, cancel and explain the blocker once. Only after revalidating that the PR is still open on the pinned SHA, report once: "Required checks passed for # at ." Never say "ready to merge", imply review approval, or merge from this monitor. Cancel after success; on the final run, stop without renewal and report an unresolved timeout once rather than claiming success. - Required-check monitors stay quiet on unchanged results and leave failed-CI reporting and remediation to existing logic. Do not duplicate task completion, PR review notifications or their retries, merged/closed PR tracking, or failed-CI triage with wakeups. Existing events remain authoritative; if they make a monitor obsolete, cancel it when tools are permitted (otherwise at its next wakeup), without repeating an already-delivered notification.