From 87427d2c0f19301ab060b90a9cf732946e79b79d Mon Sep 17 00:00:00 2001 From: "@mrubens" <2600+mrubens@users.noreply.github.com> Date: Sun, 6 Sep 2026 13:31:10 +0000 Subject: [PATCH 1/3] fix: resolve Sentry triage organization scope explicitly --- .../sentry-on-demand-dispatch.test.ts | 131 ++++++++++++++++++ .../fast-agent-integration-broker.test.ts | 74 ++++++++++ .../fast-agent-native-tool-schemas.test.ts | 37 ++++- .../__tests__/skillInvocationRouting.test.ts | 16 +++ .../skills/standard/sentry-triage/SKILL.md | 8 +- .../skills/standard/triage-sentry/SKILL.md | 4 +- 6 files changed, 264 insertions(+), 6 deletions(-) create mode 100644 apps/worker/src/mcp/roomote-mcp-server/__tests__/sentry-on-demand-dispatch.test.ts diff --git a/apps/worker/src/mcp/roomote-mcp-server/__tests__/sentry-on-demand-dispatch.test.ts b/apps/worker/src/mcp/roomote-mcp-server/__tests__/sentry-on-demand-dispatch.test.ts new file mode 100644 index 000000000..0ba0b6f87 --- /dev/null +++ b/apps/worker/src/mcp/roomote-mcp-server/__tests__/sentry-on-demand-dispatch.test.ts @@ -0,0 +1,131 @@ +import { Client } from '@modelcontextprotocol/sdk/client/index.js'; +import { InMemoryTransport } from '@modelcontextprotocol/sdk/inMemory.js'; +import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; +import { z } from 'zod'; + +import type { ToolResult } from '../types.js'; + +it('preserves required Sentry scope from discovery through registered MCP dispatch', async () => { + const originalCatalogPath = process.env.ROOMOTE_ON_DEMAND_MCP_CATALOG_PATH; + process.env.ROOMOTE_ON_DEMAND_MCP_CATALOG_PATH = '/unused/catalog.json'; + vi.resetModules(); + const integration = await import('../on-demand-integrations.js'); + const { findOnDemandIntegrationTools, callOnDemandIntegrationTool } = + integration; + const catalog = { + servers: [ + { name: 'sentry', displayName: 'Sentry', url: 'https://example.com/mcp' }, + ], + }; + const search = vi.fn(async () => ({ + content: [{ type: 'text' as const, text: '{"issues":[]}' }], + })); + const upstream = new McpServer({ name: 'sentry-fixture', version: '1.0.0' }); + upstream.registerTool( + 'search_issues', + { + inputSchema: { + organizationSlug: z.string().min(1), + query: z.string(), + projectSlugOrId: z.string().nullable().optional(), + }, + }, + search, + ); + const upstreamClient = new Client({ + name: 'upstream-test', + version: '1.0.0', + }); + const [upstreamClientTransport, upstreamServerTransport] = + InMemoryTransport.createLinkedPair(); + await upstream.connect(upstreamServerTransport); + await upstreamClient.connect(upstreamClientTransport); + + // Replace only catalog/transport boundaries; retain discovery, routing, + // member registration, wire validation, and the upstream required schema. + vi.spyOn(integration, 'loadOnDemandMcpCatalog').mockReturnValue(catalog); + vi.spyOn(integration, 'findOnDemandIntegrationTools').mockImplementation( + (servers, params) => + findOnDemandIntegrationTools( + servers, + params, + async () => (await upstreamClient.listTools()).tools, + ), + ); + vi.spyOn(integration, 'callOnDemandIntegrationTool').mockImplementation( + (servers, params) => + callOnDemandIntegrationTool( + servers, + params, + async (_server, name, args) => + (await upstreamClient.callTool({ + name, + arguments: args, + })) as ToolResult, + ), + ); + const { roomoteMcpServer } = await import('../index.js'); + const client = new Client({ name: 'member-test', version: '1.0.0' }); + const [clientTransport, serverTransport] = + InMemoryTransport.createLinkedPair(); + await roomoteMcpServer.connect(serverTransport); + await client.connect(clientTransport); + try { + expect((await client.listTools()).tools.map((tool) => tool.name)).toContain( + 'call_integration_tool', + ); + const discovered = (await client.callTool({ + name: 'find_integration_tools', + arguments: { integrationId: 'sentry', toolName: 'search_issues' }, + })) as ToolResult; + const { + tools: [tool], + } = JSON.parse(discovered.content[0]!.text!); + expect(tool.inputSchema.required).toContain('organizationSlug'); + const args = { + organizationSlug: 'example-org', + query: 'lastSeen:-24h', + projectSlugOrId: 'example-project', + }; + const call = { + integrationId: tool.integrationId, + toolName: tool.name, + args, + }; + const result = await client.callTool({ + name: 'call_integration_tool', + arguments: call, + }); + expect(result.isError).not.toBe(true); + expect(search).toHaveBeenCalledWith(args, expect.anything()); + + for (const invalidArgs of [ + { query: args.query }, + { ...args, organizationSlug: null }, + ]) { + const invalid = await client.callTool({ + name: 'call_integration_tool', + arguments: { ...call, args: invalidArgs }, + }); + expect(invalid.isError).toBe(true); + } + const unavailable = (await client.callTool({ + name: 'call_integration_tool', + arguments: { ...call, integrationId: 'not-attached' }, + })) as ToolResult; + expect(JSON.parse(unavailable.content[0]!.text!)).toMatchObject({ + success: false, + availableIntegrations: ['sentry'], + }); + expect(search).toHaveBeenCalledTimes(1); + } finally { + await client.close(); + await roomoteMcpServer.close(); + await upstreamClient.close(); + await upstream.close(); + vi.restoreAllMocks(); + if (originalCatalogPath === undefined) + delete process.env.ROOMOTE_ON_DEMAND_MCP_CATALOG_PATH; + else process.env.ROOMOTE_ON_DEMAND_MCP_CATALOG_PATH = originalCatalogPath; + } +}); diff --git a/packages/cloud-agents/src/server/fast-agent/__tests__/fast-agent-integration-broker.test.ts b/packages/cloud-agents/src/server/fast-agent/__tests__/fast-agent-integration-broker.test.ts index 519216062..1cd18e36d 100644 --- a/packages/cloud-agents/src/server/fast-agent/__tests__/fast-agent-integration-broker.test.ts +++ b/packages/cloud-agents/src/server/fast-agent/__tests__/fast-agent-integration-broker.test.ts @@ -42,6 +42,11 @@ import { clearFastAgentIntegrationToolCache, listFastAgentIntegrations as listFastAgentIntegrationsWithResolver, } from '../fast-agent-integration-broker'; +import { + CALL_INTEGRATION_TOOL_TOOL, + matchIntegrationTools, +} from '@roomote/types'; +import { z } from 'zod'; const auditContext = { userId: 'user-1', @@ -87,6 +92,75 @@ describe('fast-agent integration broker', () => { vi.useRealTimers(); }); + it('discovers and forwards required Sentry organization scope without injecting a default', async () => { + mocks.configuredServers = { + sentry: { url: 'https://api.example.com/api/mcp/sentry', headers: {} }, + }; + const inputSchema = { + type: 'object', + properties: { + organizationSlug: { type: 'string' }, + query: { type: 'string' }, + }, + required: ['organizationSlug', 'query'], + }; + mocks.listMcpTools.mockResolvedValue([ + { name: 'search_issues', inputSchema }, + ]); + mocks.callMcpTool.mockImplementation(async ({ args }) => { + z.object({ + organizationSlug: z.string().min(1), + query: z.string(), + }).parse(args); + return { issues: [] }; + }); + const available = await listFastAgentIntegrations(auditContext); + const { + tools: [tool], + } = matchIntegrationTools( + available.flatMap((integration) => + integration.tools.map((entry) => ({ + ...entry, + integrationId: integration.id, + })), + ), + { integrationId: 'sentry', toolName: 'search_issues' }, + ); + expect(tool?.inputSchema).toEqual(inputSchema); + const args = { organizationSlug: 'example-org', query: 'lastSeen:-24h' }; + const request = z.object(CALL_INTEGRATION_TOOL_TOOL.inputSchema).parse({ + integrationId: tool!.integrationId, + toolName: tool!.name, + args, + }); + await expect( + callFastAgentIntegration(auditContext, available, { + ...request, + args: request.args!, + }), + ).resolves.toEqual({ issues: [] }); + expect(mocks.callMcpTool).toHaveBeenCalledWith( + expect.objectContaining({ + url: 'https://api.example.com/api/mcp/sentry', + headers: { Authorization: 'Bearer control-plane-token' }, + args, + }), + ); + await expect( + callFastAgentIntegration(auditContext, available, { + ...request, + args: { query: args.query }, + }), + ).rejects.toThrow(); + expect(mocks.callMcpTool).toHaveBeenLastCalledWith( + expect.objectContaining({ args: { query: args.query } }), + ); + await expect( + callFastAgentIntegration(auditContext, [], { ...request, args }), + ).rejects.toThrow('not available'); + expect(mocks.callMcpTool).toHaveBeenCalledTimes(2); + }); + it('exposes the deployment GitHub App through its read-only router MCP', async () => { mocks.findGithubInstallation.mockResolvedValue({ id: 42 }); mocks.listMcpTools.mockResolvedValue([ diff --git a/packages/cloud-agents/src/server/fast-agent/__tests__/fast-agent-native-tool-schemas.test.ts b/packages/cloud-agents/src/server/fast-agent/__tests__/fast-agent-native-tool-schemas.test.ts index bf644133a..89c9cc659 100644 --- a/packages/cloud-agents/src/server/fast-agent/__tests__/fast-agent-native-tool-schemas.test.ts +++ b/packages/cloud-agents/src/server/fast-agent/__tests__/fast-agent-native-tool-schemas.test.ts @@ -4,7 +4,11 @@ import { tmpdir } from 'node:os'; import { dirname, join } from 'node:path'; import { pathToFileURL } from 'node:url'; -import { FAST_AGENT_NATIVE_TOOL_NAMES } from '@roomote/types'; +import { + CALL_INTEGRATION_TOOL_TOOL, + FAST_AGENT_NATIVE_TOOL_NAMES, +} from '@roomote/types'; +import { z } from 'zod'; import { getFastAgentNativeToolRuntime } from '../fast-agent-native-tool-bridge'; @@ -224,7 +228,7 @@ describe('Fast native tool schemas as OpenAI receives them', () => { ); await writeFile( join(workDir, 'roomote-fast-tool-bridge.js'), - 'export const invoke = async () => ({ title: "", output: "", metadata: {} });\n', + 'export const invoke = async (name, args) => ({ name, args });\n', ); await cp(sourceToolsDir, join(workDir, 'tools'), { recursive: true }); zod = await import(pathToFileURL(zodV4Entry).href); @@ -336,6 +340,35 @@ describe('Fast native tool schemas as OpenAI receives them', () => { ); }); + it('preserves required Sentry organization scope through generated tool execution and server parsing', async () => { + const callTool = tools.find( + (tool) => tool.name === FAST_AGENT_NATIVE_TOOL_NAMES.callIntegrationTool, + )!; + const request = { + integrationId: 'sentry', + toolName: 'search_issues', + args: { + organizationSlug: 'example-org', + query: 'lastSeen:-24h', + projectSlugOrId: 'example-project', + }, + }; + const parsed = zod.z + .object(callTool.args as Record) + .parse(request); + const execute = callTool.execute as ( + args: unknown, + context: unknown, + ) => Promise<{ name: string; args: unknown }>; + const forwarded = await execute(parsed, {}); + expect(forwarded.name).toBe( + FAST_AGENT_NATIVE_TOOL_NAMES.callIntegrationTool, + ); + expect( + z.object(CALL_INTEGRATION_TOOL_TOOL.inputSchema).parse(forwarded.args), + ).toEqual(request); + }); + it('rejects a bare union or object as args, the shape that broke OpenAI models', () => { const { z } = zod; const question = z.object({ id: z.string() }); diff --git a/packages/cloud-agents/src/server/workflows/__tests__/skillInvocationRouting.test.ts b/packages/cloud-agents/src/server/workflows/__tests__/skillInvocationRouting.test.ts index a333fa607..bd20f3570 100644 --- a/packages/cloud-agents/src/server/workflows/__tests__/skillInvocationRouting.test.ts +++ b/packages/cloud-agents/src/server/workflows/__tests__/skillInvocationRouting.test.ts @@ -26,6 +26,22 @@ describe('packaged skill invocation routing', () => { 'utf8', ); + it.each(['sentry-triage', 'triage-sentry'])( + '%s resolves organization scope through the available integration surface', + (skillName) => { + const skill = readPackagedSkill(skillName); + expect(skill).toContain('find_integration_tools'); + expect(skill).toContain('call_integration_tool'); + expect(skill).toContain('find_organizations'); + expect(skill).toContain('organizationSlug'); + expect(skill).toContain('inside'); + expect(skill).toContain('`args`'); + expect(skill).toContain('ambiguity'); + expect(skill).toContain('actual tool error'); + expect(skill).not.toContain('mcp__sentry__'); + }, + ); + const listBacktickMarkdownReferences = ( content: string, directoryNames: string[], diff --git a/packages/cloud-agents/src/server/workflows/skills/standard/sentry-triage/SKILL.md b/packages/cloud-agents/src/server/workflows/skills/standard/sentry-triage/SKILL.md index 1533b3303..45ae05daf 100644 --- a/packages/cloud-agents/src/server/workflows/skills/standard/sentry-triage/SKILL.md +++ b/packages/cloud-agents/src/server/workflows/skills/standard/sentry-triage/SKILL.md @@ -10,12 +10,14 @@ You are a Sentry triage specialist. Use the Sentry MCP to find the issues worth - Use the Sentry MCP already exposed in the task environment as the primary evidence source. Probe for available `mcp__sentry__*` tools before assuming access is ready, honor any project scope, scan window, Slack channel, or run mode supplied in the request, and keep scheduled/background runs read-only. This workflow should recommend code or instrumentation follow-up work; it must not plan or perform direct Sentry issue-state mutations. + Use the Sentry MCP available in the task environment as the primary evidence source. Discover its tools and confirm organization scope before assuming access is ready, honor any project scope, scan window, Slack channel, or run mode supplied in the request, and keep scheduled/background runs read-only. This workflow should recommend code or instrumentation follow-up work; it must not plan or perform direct Sentry issue-state mutations. Parse the request for `scan_window`, `project_scope`, `slack_channel_id`, `run_mode`, and trigger source. - Verify Sentry MCP readiness with a narrow read-only query. Probe the available `mcp__sentry__*` tools first, then use a minimal issue or project lookup to confirm auth and scope before scanning broadly. + When Sentry is listed as an on-demand integration, use the available `find_integration_tools` tool with its exact integration id, then `call_integration_tool` with the discovered tool name and arguments matching its advertised input schema. These wrapper names may have a `roomote_` prefix in sandboxes. Do not assume Sentry tools are directly mounted; use directly mounted tools only when actually available. + Before issue search, resolve organization scope from an explicit Sentry URL or organization supplied in the request, or discover and call the read-only organization lookup (such as `find_organizations`). Use only an accessible organization matching the requested scope. If multiple organizations could match, do not choose the first or infer an organization from a bare project slug: report the ambiguity and request the target. An explicitly all-accessible scope may cover each returned organization separately. Preserve any returned region URL when the tool schema accepts it. + Run a minimal read-only project or issue lookup to confirm the selected scope. Pass the required organization field (such as `organizationSlug`) inside the wrapper's `args` object on every scoped call, using the exact discovered field name; do not omit it, set it to null, or assume the connection injects it. Keep project filters and the scan window intact. If a call fails, distinguish missing/invalid arguments from authentication, inaccessible scope, or unavailable tools using the actual tool error. Do not diagnose an integration argument-handling bug from a failed scan alone. For scheduled runs, keep the scan task read-only even if vendor-side issue hygiene opportunities appear. Convert the strongest finding into code or instrumentation follow-up work instead of planning a direct Sentry state change. @@ -36,7 +38,7 @@ You are a Sentry triage specialist. Use the Sentry MCP to find the issues worth For each finding include project, environment, why it matters, rough Sentry evidence counts, confidence, and one recommendation: `fix-now`, `watch`, `deprioritize`, `fingerprint`, or `improve-instrumentation`. When scheduled/background context provides a `repository_scope` and the `submit_automation_work_items` tool is available, submit the strongest actionable Sentry follow-up there instead of posting those findings as plain Slack text. Submit at most one `act` work item per run, scoped to exactly one repository from `repository_scope`, and bundle multiple closely related fixable Sentry issues into that single task when they belong together. Use `actionKind: code_change_pr`. Do not submit `suggest` work items; they are rejected. When the prompt includes a `Repository environments` section, only target repositories listed there, copy the matching `targetEnvironmentId`, and do not fall back to bare-repo launches. Fold lower-confidence follow-ups or additional non-code recommendations into the single work item's investigation context or execution prompt so the later execution task can surface them. Provide an `executionPrompt` that opens with a conversational investigation sentence making it clear the task was looking through Sentry and found something worth fixing or instrumenting. That opener should briefly restate what Sentry issue or workflow was checked and what stood out, assuming the Slack reader does not already know the prior context, and it should not lead with internal confirmation language like saying the issue "was real" before it says this was a Sentry investigation. After that opener, tell the later task exactly what to change, what evidence to re-verify first, and what outcome to aim for: a reviewable PR. Prefer repository-backed fixes and observability improvements over vendor-state hygiene. When the best next step would only be a Sentry-side archive, merge, resolve, or reopen, report that recommendation in prose instead of trying to launch a direct mutation task. - Use additional read-only `mcp__sentry__*` lookups or resource fetches when they materially improve confidence about an issue's recurrence, release association, or likely owner. When the Sentry MCP does not expose enough detail directly, say so briefly and keep the recommendation scoped. + Use additional discovered read-only Sentry lookups or resource fetches when they materially improve confidence about an issue's recurrence, release association, or likely owner. When the Sentry MCP does not expose enough detail directly, say so briefly and keep the recommendation scoped. Write action-first work item titles such as `Fix ...`, `Improve fingerprinting for ...`, `Improve instrumentation for ...`, `Upload sourcemaps for ...`, or `Fix release attribution for ...`. Put `$sentry-triage`, the intended follow-up, Sentry issue URLs or IDs, project, evidence, suspected owner or stack area, the MCP tools or Sentry resources used during triage, and the verification required before editing code in `investigationContext`. Map categories by task shape: `bug` for code defects, `improvement` for instrumentation, fingerprinting, source-map, release attribution, or trace/log observability work, and `security` only when Sentry evidence shows security impact. Do not submit a launchable work item for findings whose repository ownership is unclear; when you submit a work item, fold them into its investigation context. Do not post a findings-only Slack message just to surface unclear-ownership findings on an otherwise clean run. diff --git a/packages/cloud-agents/src/server/workflows/skills/standard/triage-sentry/SKILL.md b/packages/cloud-agents/src/server/workflows/skills/standard/triage-sentry/SKILL.md index 63919790b..896852614 100644 --- a/packages/cloud-agents/src/server/workflows/skills/standard/triage-sentry/SKILL.md +++ b/packages/cloud-agents/src/server/workflows/skills/standard/triage-sentry/SKILL.md @@ -32,7 +32,9 @@ You are a Sentry triage specialist for Roomote. Find the Sentry issues materiall Probe the Sentry MCP and report auth or targeting blockers honestly. Use the Sentry MCP as the primary source for issues, events, stack traces, releases, impacted users, and issue URLs. - Probe readiness by verifying the available `mcp__sentry__*` tools and running a narrow read-only issue or project lookup instead of assuming auth and target detection are already correct. + When Sentry is listed as an on-demand integration, use the available `find_integration_tools` and `call_integration_tool` wrappers (which may have a `roomote_` prefix in sandboxes), with the exact integration id, discovered tool name, and advertised argument schema. Use directly mounted Sentry tools only when actually available. + Resolve organization scope from an explicit Sentry URL or organization in the request, or discover and call the read-only organization lookup such as `find_organizations`. Use only an accessible organization matching the requested scope. If multiple organizations could match, report the ambiguity rather than choosing the first or inferring an organization from a bare project slug. Only an explicitly all-accessible scope permits scanning each returned organization separately. + Confirm auth and scope with a narrow read-only project or issue lookup. Pass the required organization field, such as `organizationSlug`, inside `args` on every scoped wrapper call, using the exact discovered field name and any applicable returned region URL. Never omit or null the organization, assume connection-side injection, or broaden project filters to work around a failure. Diagnose argument, auth, and scope failures from the actual tool error, not from a failed scan alone. If the MCP cannot authenticate, cannot expose the needed Sentry tools, or is scoped to the wrong target, report the exact blocker. From 3461749b9e98201ad4d0a9496971673b1b759e86 Mon Sep 17 00:00:00 2001 From: "@mrubens" <2600+mrubens@users.noreply.github.com> Date: Sun, 6 Sep 2026 13:41:26 +0000 Subject: [PATCH 2/3] fix: make Sentry triage guidance outcome-driven --- .../__tests__/skillInvocationRouting.test.ts | 17 +++++++++-------- .../skills/standard/sentry-triage/SKILL.md | 9 ++++----- .../skills/standard/triage-sentry/SKILL.md | 8 ++++---- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/packages/cloud-agents/src/server/workflows/__tests__/skillInvocationRouting.test.ts b/packages/cloud-agents/src/server/workflows/__tests__/skillInvocationRouting.test.ts index bd20f3570..a5d08914c 100644 --- a/packages/cloud-agents/src/server/workflows/__tests__/skillInvocationRouting.test.ts +++ b/packages/cloud-agents/src/server/workflows/__tests__/skillInvocationRouting.test.ts @@ -27,18 +27,19 @@ describe('packaged skill invocation routing', () => { ); it.each(['sentry-triage', 'triage-sentry'])( - '%s resolves organization scope through the available integration surface', + '%s describes schema-driven Sentry discovery and scoped triage outcomes', (skillName) => { const skill = readPackagedSkill(skillName); - expect(skill).toContain('find_integration_tools'); - expect(skill).toContain('call_integration_tool'); - expect(skill).toContain('find_organizations'); - expect(skill).toContain('organizationSlug'); - expect(skill).toContain('inside'); - expect(skill).toContain('`args`'); + expect(skill).toContain('Discover the available Sentry capabilities'); + expect(skill).toContain('advertised schemas'); + expect(skill).toContain('required organization scope'); + expect(skill).toContain('last 24 hours'); + expect(skill).toContain('React, Node, and React Native'); expect(skill).toContain('ambiguity'); expect(skill).toContain('actual tool error'); - expect(skill).not.toContain('mcp__sentry__'); + expect(skill).not.toMatch( + /mcp__sentry__|find_integration_tools|call_integration_tool|find_organizations|organizationSlug|`args`/, + ); }, ); diff --git a/packages/cloud-agents/src/server/workflows/skills/standard/sentry-triage/SKILL.md b/packages/cloud-agents/src/server/workflows/skills/standard/sentry-triage/SKILL.md index 45ae05daf..cdb4411f2 100644 --- a/packages/cloud-agents/src/server/workflows/skills/standard/sentry-triage/SKILL.md +++ b/packages/cloud-agents/src/server/workflows/skills/standard/sentry-triage/SKILL.md @@ -14,17 +14,16 @@ You are a Sentry triage specialist. Use the Sentry MCP to find the issues worth - Parse the request for `scan_window`, `project_scope`, `slack_channel_id`, `run_mode`, and trigger source. - When Sentry is listed as an on-demand integration, use the available `find_integration_tools` tool with its exact integration id, then `call_integration_tool` with the discovered tool name and arguments matching its advertised input schema. These wrapper names may have a `roomote_` prefix in sandboxes. Do not assume Sentry tools are directly mounted; use directly mounted tools only when actually available. - Before issue search, resolve organization scope from an explicit Sentry URL or organization supplied in the request, or discover and call the read-only organization lookup (such as `find_organizations`). Use only an accessible organization matching the requested scope. If multiple organizations could match, do not choose the first or infer an organization from a bare project slug: report the ambiguity and request the target. An explicitly all-accessible scope may cover each returned organization separately. Preserve any returned region URL when the tool schema accepts it. - Run a minimal read-only project or issue lookup to confirm the selected scope. Pass the required organization field (such as `organizationSlug`) inside the wrapper's `args` object on every scoped call, using the exact discovered field name; do not omit it, set it to null, or assume the connection injects it. Keep project filters and the scan window intact. If a call fails, distinguish missing/invalid arguments from authentication, inaccessible scope, or unavailable tools using the actual tool error. Do not diagnose an integration argument-handling bug from a failed scan alone. + Identify the requested scan window, workloads or projects, report destination, run mode, and trigger source. Discover the available Sentry capabilities and use their advertised schemas to determine how to look up organizations, projects, and issues. + Resolve an accessible organization matching the request from supplied Sentry context or read-only discovery. If organization or project selection is ambiguous, report the ambiguity and request the target rather than guessing. Scan multiple organizations only when explicitly in scope. Confirm access with a narrow read-only lookup, supplying the required organization scope according to the advertised schemas on every scoped request; do not assume the connection injects it. Preserve region, project, and time filters. + If discovery or a lookup fails, distinguish unavailable capabilities, invalid arguments, authentication, and inaccessible scope using the actual tool error. Do not broaden access or diagnose an argument-handling bug from a failed scan alone. For scheduled runs, keep the scan task read-only even if vendor-side issue hygiene opportunities appear. Convert the strongest finding into code or instrumentation follow-up work instead of planning a direct Sentry state change. - Inspect new, regressed, trending, high-frequency, high-user-impact, and unresolved issues in the requested window. + Search the requested window, defaulting to the last 24 hours. When asked to cover React, Node, and React Native, map those workloads to accessible projects using discovered Sentry context rather than assuming project names. Inspect new, regressed, trending, high-frequency, high-user-impact, and unresolved issues, then follow the evidence, ranking, and reporting steps below. For each candidate, collect only the evidence needed to rank it from the Sentry MCP: issue ID or URL, title, project, environment, status, first/last seen, rough event and user counts, affected release, tags, and a short stack or subsystem summary. Prioritize by user impact, operational cost, frequency, severity, blast radius, and confidence that the issue is actionable for this workspace. Do not paste raw request payloads, credentials, personal data, high-volume logs, or full stack traces. diff --git a/packages/cloud-agents/src/server/workflows/skills/standard/triage-sentry/SKILL.md b/packages/cloud-agents/src/server/workflows/skills/standard/triage-sentry/SKILL.md index 896852614..bc4a14a52 100644 --- a/packages/cloud-agents/src/server/workflows/skills/standard/triage-sentry/SKILL.md +++ b/packages/cloud-agents/src/server/workflows/skills/standard/triage-sentry/SKILL.md @@ -32,10 +32,9 @@ You are a Sentry triage specialist for Roomote. Find the Sentry issues materiall Probe the Sentry MCP and report auth or targeting blockers honestly. Use the Sentry MCP as the primary source for issues, events, stack traces, releases, impacted users, and issue URLs. - When Sentry is listed as an on-demand integration, use the available `find_integration_tools` and `call_integration_tool` wrappers (which may have a `roomote_` prefix in sandboxes), with the exact integration id, discovered tool name, and advertised argument schema. Use directly mounted Sentry tools only when actually available. - Resolve organization scope from an explicit Sentry URL or organization in the request, or discover and call the read-only organization lookup such as `find_organizations`. Use only an accessible organization matching the requested scope. If multiple organizations could match, report the ambiguity rather than choosing the first or inferring an organization from a bare project slug. Only an explicitly all-accessible scope permits scanning each returned organization separately. - Confirm auth and scope with a narrow read-only project or issue lookup. Pass the required organization field, such as `organizationSlug`, inside `args` on every scoped wrapper call, using the exact discovered field name and any applicable returned region URL. Never omit or null the organization, assume connection-side injection, or broaden project filters to work around a failure. Diagnose argument, auth, and scope failures from the actual tool error, not from a failed scan alone. - If the MCP cannot authenticate, cannot expose the needed Sentry tools, or is scoped to the wrong target, report the exact blocker. + Discover the available Sentry capabilities and use their advertised schemas to determine how to look up organizations, projects, and issues. + Resolve an accessible organization matching the request from supplied Sentry context or read-only discovery. If organization or project selection is ambiguous, report the ambiguity and request the target rather than guessing. Scan multiple organizations only when explicitly in scope. Confirm access with a narrow read-only lookup, supplying the required organization scope according to the advertised schemas on every scoped request; do not assume the connection injects it. Preserve region, project, and time filters. + If discovery or a lookup fails, distinguish unavailable capabilities, invalid arguments, authentication, and inaccessible scope using the actual tool error. Do not broaden access or diagnose an argument-handling bug from a failed scan alone. @@ -43,6 +42,7 @@ You are a Sentry triage specialist for Roomote. Find the Sentry issues materiall Define the time window, environments, and issue classes to inspect. Honor an explicit time window from the prompt; otherwise scan the last 24 hours. + When asked to cover React, Node, and React Native, map those workloads to accessible projects using discovered Sentry context rather than assuming project names. Search those projects in the requested window, then follow the evidence, ranking, and reporting steps below. Honor an explicit project or project-set scope from the prompt when the user names one. Otherwise default to the Roomote project set in the target Sentry organization: `roomote`, `roomote-api`, `roomote-dispatcher`, and `roomote-worker`. Exclude `roomote-cloud` from the default scan unless the user explicitly asks to include it. Inspect issues that are new, regressed, trending, high-frequency, high-user-impact, still unresolved, or materially worse than their recent baseline. From ea4ffc93b1729107c74a78380d9abaea5c70c481 Mon Sep 17 00:00:00 2001 From: "@mrubens" <2600+mrubens@users.noreply.github.com> Date: Sun, 6 Sep 2026 13:56:41 +0000 Subject: [PATCH 3/3] fix: remove deployment assumptions from Sentry triage skills --- .../__tests__/skillInvocationRouting.test.ts | 12 ++++++--- .../skills/standard/sentry-triage/SKILL.md | 26 ++++++++----------- .../skills/standard/triage-sentry/SKILL.md | 15 +++++------ 3 files changed, 27 insertions(+), 26 deletions(-) diff --git a/packages/cloud-agents/src/server/workflows/__tests__/skillInvocationRouting.test.ts b/packages/cloud-agents/src/server/workflows/__tests__/skillInvocationRouting.test.ts index a5d08914c..9dd53f634 100644 --- a/packages/cloud-agents/src/server/workflows/__tests__/skillInvocationRouting.test.ts +++ b/packages/cloud-agents/src/server/workflows/__tests__/skillInvocationRouting.test.ts @@ -33,12 +33,18 @@ describe('packaged skill invocation routing', () => { expect(skill).toContain('Discover the available Sentry capabilities'); expect(skill).toContain('advertised schemas'); expect(skill).toContain('required organization scope'); - expect(skill).toContain('last 24 hours'); - expect(skill).toContain('React, Node, and React Native'); + expect(skill).toContain('request or automation context'); + expect(skill).toContain( + 'If the scope is unspecified, request clarification', + ); + expect(skill).toContain('requested report destination'); expect(skill).toContain('ambiguity'); expect(skill).toContain('actual tool error'); expect(skill).not.toMatch( - /mcp__sentry__|find_integration_tools|call_integration_tool|find_organizations|organizationSlug|`args`/, + /mcp__sentry__|find_integration_tools|call_integration_tool|find_organizations|organizationSlug|`args`|submit_automation_work_items|post_to_channel/, + ); + expect(skill).not.toMatch( + /Roomote|Roo Vet|roomote-|Slack|React|\bNode\b|last 24 hours|production|preview/, ); }, ); diff --git a/packages/cloud-agents/src/server/workflows/skills/standard/sentry-triage/SKILL.md b/packages/cloud-agents/src/server/workflows/skills/standard/sentry-triage/SKILL.md index cdb4411f2..a9e00b4c2 100644 --- a/packages/cloud-agents/src/server/workflows/skills/standard/sentry-triage/SKILL.md +++ b/packages/cloud-agents/src/server/workflows/skills/standard/sentry-triage/SKILL.md @@ -1,6 +1,6 @@ --- name: sentry-triage -description: Run Sentry MCP triage for a workspace, especially from scheduled background automation. Use the Sentry MCP already available in the task environment, keep scheduled runs read-only, and submit launchable Sentry follow-up actions or post concise findings to Slack when a channel is provided. +description: Run Sentry triage for the requested scope using available capabilities. Keep scans read-only, rank actionable findings, and follow the supplied reporting and follow-up requirements. --- # Sentry Triage @@ -10,11 +10,11 @@ You are a Sentry triage specialist. Use the Sentry MCP to find the issues worth - Use the Sentry MCP available in the task environment as the primary evidence source. Discover its tools and confirm organization scope before assuming access is ready, honor any project scope, scan window, Slack channel, or run mode supplied in the request, and keep scheduled/background runs read-only. This workflow should recommend code or instrumentation follow-up work; it must not plan or perform direct Sentry issue-state mutations. + Use the available Sentry connection as the primary evidence source. Discover its capabilities and confirm organization scope before assuming access is ready. Honor the project scope, scan window, report destination, and follow-up requirements supplied in the request or automation context. Keep scans read-only; recommend code or instrumentation follow-up work rather than direct Sentry issue-state mutations. - Identify the requested scan window, workloads or projects, report destination, run mode, and trigger source. Discover the available Sentry capabilities and use their advertised schemas to determine how to look up organizations, projects, and issues. + Establish the scan window, workloads or projects, environments, and report destination from the request or automation context. If the scope is unspecified, request clarification rather than choosing project, stack, or time defaults. Discover the available Sentry capabilities and use their advertised schemas to determine how to look up organizations, projects, and issues. Resolve an accessible organization matching the request from supplied Sentry context or read-only discovery. If organization or project selection is ambiguous, report the ambiguity and request the target rather than guessing. Scan multiple organizations only when explicitly in scope. Confirm access with a narrow read-only lookup, supplying the required organization scope according to the advertised schemas on every scoped request; do not assume the connection injects it. Preserve region, project, and time filters. If discovery or a lookup fails, distinguish unavailable capabilities, invalid arguments, authentication, and inaccessible scope using the actual tool error. Do not broaden access or diagnose an argument-handling bug from a failed scan alone. For scheduled runs, keep the scan task read-only even if vendor-side issue hygiene opportunities appear. Convert the strongest finding into code or instrumentation follow-up work instead of planning a direct Sentry state change. @@ -23,7 +23,7 @@ You are a Sentry triage specialist. Use the Sentry MCP to find the issues worth - Search the requested window, defaulting to the last 24 hours. When asked to cover React, Node, and React Native, map those workloads to accessible projects using discovered Sentry context rather than assuming project names. Inspect new, regressed, trending, high-frequency, high-user-impact, and unresolved issues, then follow the evidence, ranking, and reporting steps below. + Map requested workloads to accessible projects using discovered Sentry context rather than assuming project names or stacks. Search the requested window for new, regressed, trending, high-frequency, high-user-impact, and unresolved issues, then follow the evidence, ranking, and reporting steps below. For each candidate, collect only the evidence needed to rank it from the Sentry MCP: issue ID or URL, title, project, environment, status, first/last seen, rough event and user counts, affected release, tags, and a short stack or subsystem summary. Prioritize by user impact, operational cost, frequency, severity, blast radius, and confidence that the issue is actionable for this workspace. Do not paste raw request payloads, credentials, personal data, high-volume logs, or full stack traces. @@ -33,17 +33,13 @@ You are a Sentry triage specialist. Use the Sentry MCP to find the issues worth Start with the scan window, scope, overall risk, and highest-priority finding or no-op result. - Group findings by production, preview, both, or environment unclear when Sentry evidence does not expose the environment reliably. + Group findings by the environments actually present in the evidence and requested scope; mark the environment unclear when it cannot be established. For each finding include project, environment, why it matters, rough Sentry evidence counts, confidence, and one recommendation: `fix-now`, `watch`, `deprioritize`, `fingerprint`, or `improve-instrumentation`. - When scheduled/background context provides a `repository_scope` and the `submit_automation_work_items` tool is available, submit the strongest actionable Sentry follow-up there instead of posting those findings as plain Slack text. Submit at most one `act` work item per run, scoped to exactly one repository from `repository_scope`, and bundle multiple closely related fixable Sentry issues into that single task when they belong together. Use `actionKind: code_change_pr`. Do not submit `suggest` work items; they are rejected. When the prompt includes a `Repository environments` section, only target repositories listed there, copy the matching `targetEnvironmentId`, and do not fall back to bare-repo launches. Fold lower-confidence follow-ups or additional non-code recommendations into the single work item's investigation context or execution prompt so the later execution task can surface them. Provide an `executionPrompt` that opens with a conversational investigation sentence making it clear the task was looking through Sentry and found something worth fixing or instrumenting. That opener should briefly restate what Sentry issue or workflow was checked and what stood out, assuming the Slack reader does not already know the prior context, and it should not lead with internal confirmation language like saying the issue "was real" before it says this was a Sentry investigation. After that opener, tell the later task exactly what to change, what evidence to re-verify first, and what outcome to aim for: a reviewable PR. + Create follow-up work only when authorized by the request or automation context and supported by available capabilities. Honor the supplied submission limits, repository eligibility, environment requirements, and reporting policy using the advertised schemas. Do not guess repository ownership or bypass required environment coverage. Otherwise keep recommendations in the report. Prefer repository-backed fixes and observability improvements over vendor-state hygiene. When the best next step would only be a Sentry-side archive, merge, resolve, or reopen, report that recommendation in prose instead of trying to launch a direct mutation task. Use additional discovered read-only Sentry lookups or resource fetches when they materially improve confidence about an issue's recurrence, release association, or likely owner. When the Sentry MCP does not expose enough detail directly, say so briefly and keep the recommendation scoped. - Write action-first work item titles such as `Fix ...`, `Improve fingerprinting for ...`, `Improve instrumentation for ...`, `Upload sourcemaps for ...`, or `Fix release attribution for ...`. Put `$sentry-triage`, the intended follow-up, Sentry issue URLs or IDs, project, evidence, suspected owner or stack area, the MCP tools or Sentry resources used during triage, and the verification required before editing code in `investigationContext`. - Map categories by task shape: `bug` for code defects, `improvement` for instrumentation, fingerprinting, source-map, release attribution, or trace/log observability work, and `security` only when Sentry evidence shows security impact. - Do not submit a launchable work item for findings whose repository ownership is unclear; when you submit a work item, fold them into its investigation context. Do not post a findings-only Slack message just to surface unclear-ownership findings on an otherwise clean run. - If `submit_automation_work_items` succeeds, do not call `post_to_channel` and do not post a separate Slack summary. The execution task reports its own result to Slack when it finishes. - If `slack_channel_id` is present and there is a Sentry MCP setup/auth blocker, post a concise report there with `post_to_channel` so silent scheduled failures do not disappear. Keep any such report plain-language and free of raw command transcripts; exact tool usage belongs only in work item `investigationContext`. When the run is otherwise clean — no actionable findings, no configured repositories, or only non-launchable findings — stay quiet: do not post to Slack, and end with a terse internal note. A clean read-only run is not worth a channel message. - End the task response with a terse internal note when a work item was submitted or the run was clean, or the concise blocker report when a Slack post was needed. + Give follow-ups action-first titles and enough context to stand alone: the issue URLs or IDs, affected projects, evidence, likely owner, recommended change, and verification needed before editing code. + Report only to the requested report destination, if any; otherwise return the result in the current conversation. Avoid duplicate summaries when an authorized follow-up already owns reporting. Honor any supplied quiet-on-clean policy, but surface setup, authorization, or scope blockers so scheduled failures do not disappear. Keep reports concise and free of raw command transcripts or sensitive data. @@ -52,7 +48,7 @@ You are a Sentry triage specialist. Use the Sentry MCP to find the issues worth The workflow used the Sentry MCP as the primary source or reported a clear MCP/auth/setup blocker. The scan respected the requested window and project scope. Scheduled/background runs stayed read-only. -The strongest actionable scheduled finding was submitted as a single launchable Sentry follow-up work item when the tool and repository scope were available. -The final report or submitted work item was concise, prioritized, plain-language, and free of raw command transcripts, so it was safe to post in Slack. -Clean scans stayed silent in Slack; only setup/auth blockers were reported there. +Follow-up work respected the authorization, repository eligibility, and submission requirements supplied for this run. +The final report or follow-up was concise, prioritized, and free of sensitive data or raw command transcripts. +Reporting honored the requested destination and quiet-on-clean policy without hiding setup, authorization, or scope blockers. diff --git a/packages/cloud-agents/src/server/workflows/skills/standard/triage-sentry/SKILL.md b/packages/cloud-agents/src/server/workflows/skills/standard/triage-sentry/SKILL.md index bc4a14a52..cf5bf6684 100644 --- a/packages/cloud-agents/src/server/workflows/skills/standard/triage-sentry/SKILL.md +++ b/packages/cloud-agents/src/server/workflows/skills/standard/triage-sentry/SKILL.md @@ -1,21 +1,21 @@ --- name: triage-sentry version: 0.5.0 -description: 'Automation skill: Sentry issue triage workflow. Use when a task should periodically scan Roomote Sentry issues, errors, regressions, and alerts via the Sentry MCP, then rank the code or instrumentation follow-up work worth doing.' +description: 'Automation skill: Scan Sentry issues, errors, regressions, and alerts within the requested scope, then rank code or instrumentation follow-up work.' tags: - automation --- # Automation -This is an internal packaged automation skill. It ships with the worker's packaged skill catalog so automations can invoke it outside the Roomote repo. +Use this workflow for read-only Sentry triage with scope and reporting requirements supplied by the request or automation context. -You are a Sentry triage specialist for Roomote. Find the Sentry issues materially worth attention today, separate signal from noise, and turn the strongest findings into clear repository-backed follow-up recommendations. +You are a Sentry triage specialist. Find the issues materially worth attention in the requested window, separate signal from noise, and turn the strongest findings into clear repository-backed follow-up recommendations. - Run a scheduled-friendly Sentry triage workflow. Use the Sentry MCP as the evidence source, scan the requested window or the last 24 hours by default, and unless the user explicitly narrows or expands the scope, treat the Roomote project set in the target Sentry organization as in-scope by default: `roomote`, `roomote-api`, `roomote-dispatcher`, and `roomote-worker`. Do not include `roomote-cloud` in the default scan. Treat production and preview as separately important, produce a concise prioritized report, and stay read-only. + Use the available Sentry connection as the evidence source, scan the requested projects and time window, produce a concise prioritized report, and stay read-only. Discover scope from the request or automation context rather than assuming an organization's projects, stacks, or environments. @@ -41,10 +41,8 @@ You are a Sentry triage specialist for Roomote. Find the Sentry issues materiall Set scan scope Define the time window, environments, and issue classes to inspect. - Honor an explicit time window from the prompt; otherwise scan the last 24 hours. - When asked to cover React, Node, and React Native, map those workloads to accessible projects using discovered Sentry context rather than assuming project names. Search those projects in the requested window, then follow the evidence, ranking, and reporting steps below. - Honor an explicit project or project-set scope from the prompt when the user names one. Otherwise default to the Roomote project set in the target Sentry organization: `roomote`, `roomote-api`, `roomote-dispatcher`, and `roomote-worker`. - Exclude `roomote-cloud` from the default scan unless the user explicitly asks to include it. + Establish the scan window, workloads or projects, and environments from the request or automation context. If the scope is unspecified, request clarification rather than choosing project, stack, or time defaults. + Map requested workloads to accessible projects using discovered Sentry context rather than assuming project names or stacks. Search those projects in the requested window, then follow the evidence, ranking, and reporting steps below. Inspect issues that are new, regressed, trending, high-frequency, high-user-impact, still unresolved, or materially worse than their recent baseline. @@ -84,6 +82,7 @@ You are a Sentry triage specialist for Roomote. Find the Sentry issues materiall For each finding include project, environment, why it matters, rough evidence counts, confidence, and one recommendation. If a finding maps clearly to a repository-backed change, say what to change and what to verify first. Call out any setup, auth, or evidence gaps that lowered confidence. + Report only to the requested report destination, if any; otherwise return the result in the current conversation. Honor supplied reporting and follow-up requirements without guessing repository ownership or authorization, and avoid duplicate summaries. Do not hide setup, authorization, or scope blockers.