diff --git a/README.md b/README.md index 2ab2b04b..a97d67c8 100644 --- a/README.md +++ b/README.md @@ -1211,3 +1211,5 @@ Use the existing `firecrawl_feedback` tool with `endpoint: "alexandria"`: ``` This uses authenticated `POST /v2/feedback`, without a job ID, job-age deadline, or credit refund. Optional `providerFeedback` and `capabilityFeedback` arrays describe coverage gaps and execution issues; the tool schema lists supported issue values. A `new_capability_request` requires `requestedFunctionality`; `missing_capability` (the provider exists but lacks the capability) does not. Existing feedback opt-out and authentication controls apply. + +Agents are pointed at this loop from three places: the server instructions, the `firecrawl_scrape` and `firecrawl_find_tools` descriptions, and a `feedbackTool` object attached to every Alexandria execution and discovery result (with the tool name and a skeleton of the arguments). The hint is omitted for Firecrawl-internal calls such as `bash` and when `firecrawl_feedback` is not registered (`FIRECRAWL_NO_ENDPOINT_FEEDBACK` or keyless startup). diff --git a/package.json b/package.json index 077391a6..34deade1 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "firecrawl-mcp", - "version": "3.25.0", + "version": "3.25.1", "description": "MCP server for Firecrawl — search, scrape, and interact with the web, and search scientific papers. Supports both cloud and self-hosted instances. Features include web search, scraping, page interaction, batch processing, LLM-powered content analysis, and research paper search over biomedical and arXiv literature (PubMed, bioRxiv, medRxiv, arXiv) with citation-graph expansion and full-text reading.", "type": "module", "mcpName": "io.github.firecrawl/firecrawl-mcp-server", diff --git a/src/alexandria-feedback.ts b/src/alexandria-feedback.ts index 2819a2a8..6cf49458 100644 --- a/src/alexandria-feedback.ts +++ b/src/alexandria-feedback.ts @@ -56,6 +56,52 @@ export const alexandriaFeedbackFields = { .optional(), }; +/** + * Shared by the server instructions and the Alexandria tool descriptions so an + * agent that only reads one of them still learns the feedback loop exists. + * Wording is checked by scripts/agent-metadata-policy.mjs. + */ +export const ALEXANDRIA_FEEDBACK_GUIDANCE = + 'After an Alexandria task, whether a capability ran or discovery found nothing for the website, call firecrawl_feedback once per website with endpoint "alexandria": requestedWebsite (the url the user needed data from and the requestedFunctionality they needed), a rating, a rationale from observed results, and any providerFeedback or capabilityFeedback gaps. It is free, needs no job ID, has no deadline, and follows the answer rather than replacing it.'; + +/** Appended to Alexandria results so the pointer travels with the data the agent is reading. */ +export const ALEXANDRIA_FEEDBACK_HINT = { + name: 'firecrawl_feedback', + when: 'Once per website after the task is complete, including when no provider covered the site. Free; no job ID or deadline.', + arguments: { + endpoint: 'alexandria', + rating: '', + requestedWebsite: { + url: '', + requestedFunctionality: '', + }, + rationale: '', + providerFeedback: + '', + capabilityFeedback: + '', + }, +} as const; + +type AlexandriaCall = { provider: string; capability: string }; + +/** Discovery and provider executions get the hint; Firecrawl-internal calls such as bash or terms do not. */ +export function alexandriaCallsWarrantFeedback( + calls: AlexandriaCall[] +): boolean { + return calls.some( + (call) => call.provider !== 'firecrawl' || call.capability === 'find-tools' + ); +} + +export function withAlexandriaFeedbackHint( + envelope: T, + enabled: boolean +): T { + if (!enabled || !envelope || typeof envelope !== 'object') return envelope; + return { ...envelope, feedbackTool: ALEXANDRIA_FEEDBACK_HINT }; +} + export const alexandriaSessionFeedbackSchema = z.strictObject({ endpoint: z.literal('alexandria'), rating: z.enum(['good', 'bad', 'partial']), diff --git a/src/alexandria-output.ts b/src/alexandria-output.ts index 4687fc0f..a368c8ff 100644 --- a/src/alexandria-output.ts +++ b/src/alexandria-output.ts @@ -7,9 +7,11 @@ export async function alexandriaOutput( payload: any, calls: Call[], post: Post, - recoveryRequestId: string + recoveryRequestId: string, + // Merged into whichever envelope is returned, inline or retained. + extras: Record = {} ): Promise { - const serialized = JSON.stringify(payload); + const serialized = JSON.stringify({ ...payload, ...extras }); const responseBytes = Buffer.byteLength(serialized, 'utf8'); const estimatedTokens = Math.ceil(responseBytes / 4); const items = payload?.data?.alexandria; @@ -83,6 +85,7 @@ export async function alexandriaOutput( }, }, }, + ...extras, }); } catch { return serialized; diff --git a/src/alexandria.ts b/src/alexandria.ts index 14abb093..51b7a9eb 100644 --- a/src/alexandria.ts +++ b/src/alexandria.ts @@ -1,4 +1,5 @@ import { z } from 'zod'; +import { ALEXANDRIA_FEEDBACK_GUIDANCE } from './alexandria-feedback.js'; const catalogueTypes = ['alexandria', 'exchange'] as const; export const searchSourceSchema = z.union([ @@ -67,7 +68,8 @@ export const ALEXANDRIA_INSTRUCTIONS = 'Use sources: ["alexandria"] for semantic tools only, sources: ["web"] for web only, or sources: ["web"], domainTools: true for web plus domain tools. domainTools: false disables domain matching. Results in data.tools describe available tools, not executed data. Search defaults to toolDetail: "compact", returning only provider, capability and description; "summary" adds metadata and navigation. Inspect selected tools with firecrawl_find_tools using their providers and capabilities plus expand:["options","response"]; request examples only when the input shape is unclear. Batch related contract inspections and reuse complete contracts. Alternatively set toolDetail: \"full\" for contracts upfront. ' + 'On the full MCP surface, firecrawl_find_tools supports semantic query lookup and is the list equivalent: {} lists categories; {categories:[""]} lists providers; {providers:[""]} lists compact tools; adding capabilities:[""] expands the selected contract. Avoid expanding the entire catalogue. ' + 'Read required inputs and requiresOneOf groups (at least one member per group), example.request/example.response when present, and response.key in the selected contract. Do not assume records is the result key. Follow the declared pagination input and response cursor, preserving filters; catalogue next is separate from provider pagination. ' + - 'Execute through firecrawl_scrape with alexandria:{provider,capability,options}. Search scrapeOptions fetches web pages, never provider tools. Use web results when sufficient, tools when they offer a direct route to deeper data.'; + 'Execute through firecrawl_scrape with alexandria:{provider,capability,options}. Search scrapeOptions fetches web pages, never provider tools. Use web results when sufficient, tools when they offer a direct route to deeper data. ' + + ALEXANDRIA_FEEDBACK_GUIDANCE; export function findToolsOptions(args: z.infer) { const level = args.level ?? (args.query || args.capabilities?.length || args.providers?.length || args.groups?.length || args.urls?.length ? 'tools' : args.categories?.length ? 'providers' : 'categories'); diff --git a/src/index.ts b/src/index.ts index 371ad130..c65e742e 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,5 +1,12 @@ #!/usr/bin/env node -import { alexandriaFeedbackFields, alexandriaSessionFeedbackSchema } from './alexandria-feedback.js'; +import { + ALEXANDRIA_FEEDBACK_GUIDANCE, + ALEXANDRIA_FEEDBACK_HINT, + alexandriaCallsWarrantFeedback, + alexandriaFeedbackFields, + alexandriaSessionFeedbackSchema, + withAlexandriaFeedbackHint, +} from './alexandria-feedback.js'; import FirecrawlApp from 'firecrawl'; import dotenv from 'dotenv'; import { FastMCP, type Logger, UserError } from 'fastmcp'; @@ -1202,7 +1209,7 @@ const openAiAppsChallengeToken = normalizeHeader( ); const FULL_PROFILE_INSTRUCTIONS = - `Execution with firecrawl_scrape alexandria uses requestId; reuse the returned ID for retries of the same payload, never a new ID to bypass a pending or uncertain 409. Firecrawl provides web search, page retrieval, site URL discovery, multi-page collection, structured page data, monitoring, and multi-source research that returns structured data. Match the requested operation to the tool boundary: firecrawl_scrape retrieves one supplied page and can return JSON matching a supplied schema, firecrawl_map enumerates URLs under a site without retrieving their content, and firecrawl_agent runs multi-source research and returns structured data when the URLs are not known or the answer spans several sites (an entity plus its fields, a list, a dataset); its result is read with firecrawl_agent_status. For biomedical, life-science, clinical, or arXiv literature, the firecrawl_research_* tools search a paper index of abstracts and full text; firecrawl_search with categories: ["research"] is a website filter over ordinary web results and reaches different sources. For a programming question — code behaviour, a library or framework, an API contract, an error message, or a known bug — firecrawl_developer_search (or firecrawl_search with categories: ["developer"]) searches an index of repositories, GitHub issues, merged pull requests, READMEs, and curated documentation sites. For structured records, filterable listings, transcripts, or datasets, first check firecrawl_find_tools for a suitable workflow or data provider. Use firecrawl_search for ordinary web research and firecrawl_scrape for a known page. If discovery has no suitable match, continue with web search or firecrawl_agent. firecrawl_search with sources: [{type: "alexandria"}] returns compact tool summaries in data.tools; toolDetail: "full" includes contracts, firecrawl_find_tools starts with categories, lists providers, then compact tools, and expands the selected full contract, and firecrawl_scrape with alexandria: [{provider, capability, options}] executes up to ten capabilities and returns their results. Alexandria access needs an API key on a team with it enabled. A terms-gated Alexandria provider fails with code THIRD_PARTY_DATA_TERMS_REQUIRED and a requiresAction.url. Follow the returned terms/show and terms/accept calls through firecrawl_scrape; acceptance requires explicit user authorization for the reviewed version and digest and confirmed:true. Otherwise direct an organization admin to the dashboard URL. Do not repeat successful provider calls just because the client could not display their output. Provide only the required inputs and account for stated network or external side effects.`; + `Execution with firecrawl_scrape alexandria uses requestId; reuse the returned ID for retries of the same payload, never a new ID to bypass a pending or uncertain 409. Firecrawl provides web search, page retrieval, site URL discovery, multi-page collection, structured page data, monitoring, and multi-source research that returns structured data. Match the requested operation to the tool boundary: firecrawl_scrape retrieves one supplied page and can return JSON matching a supplied schema, firecrawl_map enumerates URLs under a site without retrieving their content, and firecrawl_agent runs multi-source research and returns structured data when the URLs are not known or the answer spans several sites (an entity plus its fields, a list, a dataset); its result is read with firecrawl_agent_status. For biomedical, life-science, clinical, or arXiv literature, the firecrawl_research_* tools search a paper index of abstracts and full text; firecrawl_search with categories: ["research"] is a website filter over ordinary web results and reaches different sources. For a programming question — code behaviour, a library or framework, an API contract, an error message, or a known bug — firecrawl_developer_search (or firecrawl_search with categories: ["developer"]) searches an index of repositories, GitHub issues, merged pull requests, READMEs, and curated documentation sites. For structured records, filterable listings, transcripts, or datasets, first check firecrawl_find_tools for a suitable workflow or data provider. Use firecrawl_search for ordinary web research and firecrawl_scrape for a known page. If discovery has no suitable match, continue with web search or firecrawl_agent. firecrawl_search with sources: [{type: "alexandria"}] returns compact tool summaries in data.tools; toolDetail: "full" includes contracts, firecrawl_find_tools starts with categories, lists providers, then compact tools, and expands the selected full contract, and firecrawl_scrape with alexandria: [{provider, capability, options}] executes up to ten capabilities and returns their results. Alexandria access needs an API key on a team with it enabled. A terms-gated Alexandria provider fails with code THIRD_PARTY_DATA_TERMS_REQUIRED and a requiresAction.url. Follow the returned terms/show and terms/accept calls through firecrawl_scrape; acceptance requires explicit user authorization for the reviewed version and digest and confirmed:true. Otherwise direct an organization admin to the dashboard URL. Do not repeat successful provider calls just because the client could not display their output. Provide only the required inputs and account for stated network or external side effects. ${ALEXANDRIA_FEEDBACK_GUIDANCE}`; const KEYLESS_PROFILE_INSTRUCTIONS = `Hosted keyless sessions expose firecrawl_search, firecrawl_scrape, and firecrawl_parse with usage limits. firecrawl_search searches the web. For programming questions, firecrawl_search with categories: ["developer"] searches indexed repositories, GitHub issues, merged pull requests, repository READMEs, and curated documentation sites. For biomedical, life-science, clinical, or arXiv literature, firecrawl_search with categories: ["research"] filters ordinary web results to research-affiliated websites. firecrawl_scrape retrieves one supplied page and can return JSON matching a supplied schema. firecrawl_parse processes supported local files through its two-phase upload flow. An Authorization bearer API key can provide higher usage limits and expose additional tools, subject to plan, deployment, and team policy, including firecrawl_map for site URL discovery, firecrawl_agent and firecrawl_agent_status for multi-source research that returns structured data when the URLs are not known, firecrawl_research_* for paper-index and repository research, and firecrawl_find_tools as the progressive Alexandria catalogue lookup alongside the Alexandria options of firecrawl_search and firecrawl_scrape for catalogued data providers.`; // The search surface exposes web/developer/research search only. Its instructions @@ -2437,7 +2444,7 @@ Firecrawl may reuse recently indexed content instead of refetching the page, and Returns the selected content formats and page metadata. Authenticated responses can include a \`metadata.scrapeId\` for optional scrape feedback. -Alexandria mode: pass \`alexandria\` (one \`{provider, capability, options}\` object or an array of 1-10) instead of \`url\` to execute catalogued Alexandria capabilities found through \`firecrawl_search\` sources \`alexandria\` or \`firecrawl_find_tools\`. The optional requestId identifies one logical execution: reuse the returned ID for retries of the identical payload, never a new ID to bypass pending or uncertain execution. Each call may include version to pin a published workflow; omitting it uses latest. Only timeout also applies at the top level in this mode. Returns per-capability results in \`data.alexandria\`, including \`data\`, \`records\`, or an \`error\` with a code. Check each item for errors even when the outer response is successful. Alexandria needs an API key on a team with Alexandria enabled. +Alexandria mode: pass \`alexandria\` (one \`{provider, capability, options}\` object or an array of 1-10) instead of \`url\` to execute catalogued Alexandria capabilities found through \`firecrawl_search\` sources \`alexandria\` or \`firecrawl_find_tools\`. The optional requestId identifies one logical execution: reuse the returned ID for retries of the identical payload, never a new ID to bypass pending or uncertain execution. Each call may include version to pin a published workflow; omitting it uses latest. Only timeout also applies at the top level in this mode. Returns per-capability results in \`data.alexandria\`, including \`data\`, \`records\`, or an \`error\` with a code. Check each item for errors even when the outer response is successful. Alexandria needs an API key on a team with Alexandria enabled. Alexandria results include a \`feedbackTool\` pointer: after the task, report how the catalogue served the website through \`firecrawl_feedback\` with endpoint \`alexandria\` (free, no job ID). For potentially large workflow results, supply and preserve a top-level requestId before execution. If a response provides nextTool, follow its instructions to access the result without repeating a successful provider call. @@ -2663,9 +2670,10 @@ async function executeExchangeCalls( ), { tool: 'firecrawl_scrape', requestId, providers: (Array.isArray(alexandria) ? alexandria : [alexandria]).map(call => call.provider) } ); + const calls = Array.isArray(alexandria) ? alexandria : [alexandria]; return alexandriaOutput( { ...(response?.data ?? {}), requestId }, - Array.isArray(alexandria) ? alexandria : [alexandria], + calls, async (body, id) => { const result = await (client as any).http.post( '/v2/scrape', @@ -2674,7 +2682,10 @@ async function executeExchangeCalls( ); return result?.data; }, - randomUUID() + randomUUID(), + alexandriaFeedbackAvailable() && alexandriaCallsWarrantFeedback(calls) + ? { feedbackTool: ALEXANDRIA_FEEDBACK_HINT } + : {} ); } catch (error) { if ((error as any)?.response?.status === 401) throw error; @@ -2695,7 +2706,7 @@ server.addTool({ idempotentHint: true, }, description: - 'Find ready-made workflows, data APIs, and indexes for structured data beyond a single webpage. Check here first for filterable records, listings, transcripts, or datasets. Use query for semantic discovery or urls to find tools for a website. With no arguments, browse categories, then providers and tools. Inspect a selected contract before executing through firecrawl_scrape; reuse contracts already returned. Follow nextTool for further discovery or pagination. Discovery does not execute providers. Use firecrawl_search when you also need web results.', + 'Find ready-made workflows, data APIs, and indexes for structured data beyond a single webpage. Check here first for filterable records, listings, transcripts, or datasets. Use query for semantic discovery or urls to find tools for a website. With no arguments, browse categories, then providers and tools. Inspect a selected contract before executing through firecrawl_scrape; reuse contracts already returned. Follow nextTool for further discovery or pagination. Discovery does not execute providers. Use firecrawl_search when you also need web results. Results include a feedbackTool pointer: after the task, report how the catalogue served the website through firecrawl_feedback with endpoint alexandria, including when nothing covered it (free, no job ID).', parameters: findToolsSchema, execute: async (args, { session, client: mcpClient }) => { const origin = requestOrigin(mcpClient, session); @@ -2719,7 +2730,7 @@ server.addTool({ })); const page: any = { level: 'categories', items, total: rows.length }; if (offset + options.limit < rows.length) page.nextTool = { name: 'firecrawl_find_tools', arguments: { level: 'categories', limit: options.limit, offset: offset + options.limit } }; - return compactText(withFindToolsNavigation({ success: true, data: { creditsCost: 0, alexandria: [{ provider: 'firecrawl', capability: 'find-tools', creditsCost: 0, data: page }] } })); + return compactText(withAlexandriaFeedbackHint(withFindToolsNavigation({ success: true, data: { creditsCost: 0, alexandria: [{ provider: 'firecrawl', capability: 'find-tools', creditsCost: 0, data: page }] } }), alexandriaFeedbackAvailable())); } const result = await executeExchangeCalls(session, { provider: 'firecrawl', capability: 'find-tools', options }, undefined, undefined, origin); return compactText(withFindToolsNavigation(JSON.parse(result))); @@ -3147,7 +3158,12 @@ if (ENDPOINT_FEEDBACK_DISABLED) { ); } -if (!ENDPOINT_FEEDBACK_DISABLED && !isLocalKeylessStartup()) { +/** Whether firecrawl_feedback is registered on this process, so Alexandria results only point at a tool that exists. */ +function alexandriaFeedbackAvailable(): boolean { + return !ENDPOINT_FEEDBACK_DISABLED && !isLocalKeylessStartup(); +} + +if (alexandriaFeedbackAvailable()) { server.addTool({ name: 'firecrawl_feedback', annotations: { diff --git a/tests/mcp-alexandria-feedback-hints.test.mjs b/tests/mcp-alexandria-feedback-hints.test.mjs new file mode 100644 index 00000000..57dea740 --- /dev/null +++ b/tests/mcp-alexandria-feedback-hints.test.mjs @@ -0,0 +1,75 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { startStdio, startStdioWithApi, toolText } from './helpers/exchange-mcp.mjs'; +import { EXCHANGE_CALL, startFakeExchangeApi } from './helpers/exchange-api.mjs'; + +const largeAlexandria = () => ({ + success: true, + data: { creditsCost: 5, alexandria: [{ ...EXCHANGE_CALL, data: { text: 'x'.repeat(90_000) } }] }, +}); + +function assertFeedbackHint(payload) { + assert.equal(payload.feedbackTool?.name, 'firecrawl_feedback'); + assert.equal(payload.feedbackTool.arguments.endpoint, 'alexandria'); + assert.match(payload.feedbackTool.arguments.requestedWebsite.url, /website/i); + assert.match(payload.feedbackTool.when, /once per website/i); +} + +test('server instructions and Alexandria tool descriptions point at firecrawl_feedback', async (t) => { + const { client, init } = await startStdioWithApi(t); + assert.match(init.instructions, /call firecrawl_feedback once per website with endpoint "alexandria"/); + assert.match(init.instructions, /whether a capability ran or discovery found nothing for the website/); + const { tools } = await client.request('tools/list', {}); + const byName = new Map(tools.map((tool) => [tool.name, tool])); + assert(byName.has('firecrawl_feedback')); + assert.match(byName.get('firecrawl_scrape').description, /feedbackTool.*firecrawl_feedback.*endpoint `alexandria`/s); + assert.match(byName.get('firecrawl_find_tools').description, /feedbackTool.*firecrawl_feedback.*including when nothing covered it/s); + assert.match(byName.get('firecrawl_search').description, /call firecrawl_feedback once per website/); +}); + +test('Alexandria executions and discovery results carry the feedback hint; utility calls do not', async (t) => { + const { client } = await startStdioWithApi(t); + const call = (name, args) => client.request('tools/call', { name, arguments: args }); + + const executed = toolText(await call('firecrawl_scrape', { alexandria: EXCHANGE_CALL })); + assertFeedbackHint(executed); + assert.equal(executed.success, true); + + const categories = toolText(await call('firecrawl_find_tools', { limit: 1 })); + assertFeedbackHint(categories); + assert.equal(categories.data.alexandria[0].data.level, 'categories'); + + const tools = toolText(await call('firecrawl_find_tools', { providers: ['particle'] })); + assertFeedbackHint(tools); + + const bash = toolText( + await call('firecrawl_scrape', { + alexandria: { provider: 'firecrawl', capability: 'bash', options: { workspaceId: 'existing', command: 'cat response.json' } }, + }) + ); + assert.equal(bash.feedbackTool, undefined); +}); + +test('retained Alexandria results keep the feedback hint next to nextTool', async (t) => { + const { client } = await startStdioWithApi(t, { largeResult: largeAlexandria(), bashRecovery: 'available' }); + const result = toolText(await client.request('tools/call', { name: 'firecrawl_scrape', arguments: { alexandria: EXCHANGE_CALL } })); + assert.equal(result.delivery, 'retained'); + assert.equal(result.nextTool.name, 'firecrawl_scrape'); + assertFeedbackHint(result); +}); + +test('no feedback hint is attached when the feedback tool is disabled by environment', async (t) => { + const api = await startFakeExchangeApi({}); + t.after(() => api.close()); + const { client } = await startStdio(t, { + FIRECRAWL_API_KEY: 'fc-exchange-test', + FIRECRAWL_API_URL: api.url, + FIRECRAWL_NO_ENDPOINT_FEEDBACK: '1', + }); + const { tools } = await client.request('tools/list', {}); + assert.equal(tools.some((tool) => tool.name === 'firecrawl_feedback'), false); + const executed = toolText(await client.request('tools/call', { name: 'firecrawl_scrape', arguments: { alexandria: EXCHANGE_CALL } })); + assert.equal(executed.feedbackTool, undefined); + const categories = toolText(await client.request('tools/call', { name: 'firecrawl_find_tools', arguments: { limit: 1 } })); + assert.equal(categories.feedbackTool, undefined); +});