diff --git a/package.json b/package.json index a4458d192e..6b6d27c6e7 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "firecrawl-cli", - "version": "1.24.3", + "version": "1.24.4", "publishConfig": { "tag": "latest" }, diff --git a/src/__tests__/alexandria-beta.test.ts b/src/__tests__/alexandria-beta.test.ts index d96a37999f..57051f039a 100644 --- a/src/__tests__/alexandria-beta.test.ts +++ b/src/__tests__/alexandria-beta.test.ts @@ -724,6 +724,174 @@ it('find-tools and explicit meta-tool execution use the same Scrape request', as expect(requests[0].url).toBe('/v2/scrape'); }); +it('accepts the HTTP contract object form for --alexandria and rejects malformed addresses locally', async () => { + const object = await cli([ + 'scrape', + '--alexandria', + '{"provider":"benzinga","capability":"news/search","options":{"pageSize":1}}', + '--request-id', + 'object-1', + ]); + expect(object.code).toBe(0); + const bare = await cli([ + 'scrape', + '--alexandria', + 'benzinga/news/search', + '--options', + '{"pageSize":1}', + '--request-id', + 'object-1', + ]); + expect(bare.code).toBe(0); + expect(requests).toHaveLength(2); + expect(requests[0]).toEqual(requests[1]); + expect(requests[0].body.alexandria).toEqual([ + { + provider: 'benzinga', + capability: 'news/search', + options: { pageSize: 1 }, + }, + ]); + const separate = await cli([ + 'scrape', + '--alexandria', + '{"provider":"benzinga","capability":"news/search"}', + '--options', + '{"pageSize":2}', + ]); + expect(separate.code).toBe(0); + expect(requests[2].body.alexandria[0].options).toEqual({ pageSize: 2 }); + for (const address of [ + '{"provider":"benzinga"}', + '{"provider":"benzinga","capability":"news/search","extra":1}', + '{not json', + 'benzinga', + ]) { + const result = await cli(['scrape', '--alexandria', address]); + expect(result.code).toBe(1); + expect(result.stderr).toContain('provider/capability'); + } + const both = await cli([ + 'scrape', + '--alexandria', + '{"provider":"benzinga","capability":"news/search","options":{}}', + '--options', + '{}', + ]); + expect(both.code).toBe(1); + expect(both.stderr).toContain('not both'); + expect(requests).toHaveLength(3); +}); + +it('accepts the comma-separated expand form documented in find-tools --help', async () => { + const help = await cli(['find-tools', '--help']); + expect(help.stdout.replace(/\s+/g, ' ')).toContain( + 'expand: ["options","response","examples"]' + ); + expect( + ( + await cli([ + 'find-tools', + '--options', + '{"providers":"fred","expand":"options, response,examples"}', + ]) + ).code + ).toBe(0); + expect( + ( + await cli([ + 'find-tools', + '--options', + '{"providers":["fred"],"expand":["options","response","examples"]}', + ]) + ).code + ).toBe(0); + expect( + ( + await cli([ + 'scrape', + '--alexandria', + 'firecrawl/find-tools', + '--options', + '{"providers":"fred","expand":"options,response,examples"}', + ]) + ).code + ).toBe(0); + expect(requests).toHaveLength(3); + for (const request of requests) + expect(request.body.alexandria[0].options).toEqual({ + providers: ['fred'], + expand: ['options', 'response', 'examples'], + }); +}); + +it('never comma-splits urls, which may legally contain commas', async () => { + const url = 'https://example.com/items,23?ids=1,2'; + expect( + ( + await cli([ + 'find-tools', + '--options', + JSON.stringify({ urls: url, providers: 'fred,benzinga' }), + ]) + ).code + ).toBe(0); + expect( + ( + await cli([ + 'find-tools', + '--request', + JSON.stringify({ + provider: 'firecrawl', + capability: 'find-tools', + options: { urls: url, providers: 'fred,benzinga' }, + }), + ]) + ).code + ).toBe(0); + expect( + ( + await cli([ + 'scrape', + '--alexandria', + 'firecrawl/find-tools', + '--options', + JSON.stringify({ urls: [url], providers: 'fred,benzinga' }), + ]) + ).code + ).toBe(0); + expect(requests).toHaveLength(3); + for (const request of requests) + expect(request.body.alexandria[0].options).toEqual({ + urls: [url], + providers: ['fred', 'benzinga'], + }); +}); + +it('trims whitespace around the slash in bare addresses and rejects empty parts', async () => { + const padded = await cli([ + 'scrape', + '--alexandria', + ' benzinga / news/search ', + '--options', + '{"pageSize":1}', + ]); + expect(padded.code).toBe(0); + expect(requests[0].body.alexandria).toEqual([ + { + provider: 'benzinga', + capability: 'news/search', + options: { pageSize: 1 }, + }, + ]); + for (const address of ['benzinga/ ', ' /news', '/', 'benzinga/']) { + const result = await cli(['scrape', '--alexandria', address]); + expect(result.code).toBe(1); + expect(result.stderr).toContain('provider/capability'); + } + expect(requests).toHaveLength(1); +}); + it('discovers a known URL and follows its returned meta-tool request without executing providers', async () => { const next = { provider: 'firecrawl', diff --git a/src/commands/alexandria.ts b/src/commands/alexandria.ts index 243218f522..b5cc9e72bc 100644 --- a/src/commands/alexandria.ts +++ b/src/commands/alexandria.ts @@ -32,6 +32,78 @@ export function parseToolOptions(raw = '{}'): Record { return value; } +const FIND_TOOLS_LIST_SELECTORS = [ + 'providers', + 'categories', + 'groups', + 'capabilities', + 'expand', +]; + +/** + * Find Tools list selectors take arrays; also accept the comma-separated form. + * `urls` is never comma-split (a comma is legal inside a URL); a single string is wrapped. + */ +export function normalizeFindToolsOptions( + options: Record +): Record { + const next = { ...options }; + for (const key of FIND_TOOLS_LIST_SELECTORS) { + const value = next[key]; + if (typeof value === 'string') + next[key] = value + .split(',') + .map((part) => part.trim()) + .filter(Boolean); + } + if (typeof next.urls === 'string') next.urls = [next.urls.trim()]; + return next; +} + +const ADDRESS_HINT = + 'Use a provider/capability address, or a JSON object {"provider":"...","capability":"...","options":{...}}.'; + +/** Accept both the bare `provider/capability` form and the HTTP contract's object form. */ +function parseAddress(address: string): AlexandriaCall { + const trimmed = address.trim(); + if (trimmed.startsWith('{')) { + let value: any; + try { + value = JSON.parse(trimmed); + } catch { + throw new Error(ADDRESS_HINT); + } + if ( + !value || + typeof value !== 'object' || + Array.isArray(value) || + typeof value.provider !== 'string' || + !value.provider.trim() || + typeof value.capability !== 'string' || + !value.capability.trim() || + (value.options !== undefined && + (!value.options || + typeof value.options !== 'object' || + Array.isArray(value.options))) || + Object.keys(value).some( + (key) => !['provider', 'capability', 'options'].includes(key) + ) + ) + throw new Error(ADDRESS_HINT); + return { + provider: value.provider.trim(), + capability: value.capability.trim(), + ...(value.options !== undefined && { options: value.options }), + }; + } + const slash = trimmed.indexOf('/'); + if (slash < 0) throw new Error(ADDRESS_HINT); + const provider = trimmed.slice(0, slash).trim(); + const capability = trimmed.slice(slash + 1).trim(); + if (!provider || !capability) throw new Error(ADDRESS_HINT); + return { provider, capability }; +} + export function buildCalls(addresses: string[], values: string[] = []): Call[] { if ( !addresses.length || @@ -42,13 +114,19 @@ export function buildCalls(addresses: string[], values: string[] = []): Call[] { 'Provide 1-10 capabilities, with at most one --options value per capability.' ); return addresses.map((address, i) => { - const slash = address.indexOf('/'); - if (slash < 1 || slash === address.length - 1) - throw new Error('Use a provider/capability address.'); + const call = parseAddress(address); + if (call.options !== undefined && values[i] !== undefined) + throw new Error( + `Provide options for ${call.provider}/${call.capability} either inside the object or with --options, not both.` + ); + const options = call.options ?? parseToolOptions(values[i]); return { - provider: address.slice(0, slash), - capability: address.slice(slash + 1), - options: parseToolOptions(values[i]), + provider: call.provider, + capability: call.capability, + options: + call.provider === 'firecrawl' && call.capability === 'find-tools' + ? normalizeFindToolsOptions(options) + : options, }; }); } @@ -187,7 +265,9 @@ export function parseFindToolsRequest(raw: string): Call { return { provider: 'firecrawl', capability: 'find-tools', - options: parseToolOptions(JSON.stringify(next.options)), + options: normalizeFindToolsOptions( + parseToolOptions(JSON.stringify(next.options)) + ), }; } @@ -199,7 +279,7 @@ export function createFindToolsCommand(): Command { .argument('[urls...]', 'Known HTTP(S) URLs to find tools for') .option( '--options ', - 'Discovery options: query, urls, providers, categories, groups, capabilities; level: providers|groups|tools; limit: 1-100; expand: options,response,examples. Use provider and capability IDs returned by discovery.' + 'Discovery options: query, urls, providers, categories, groups, capabilities; level: providers|groups|tools; limit: 1-100; expand: ["options","response","examples"] (a comma-separated string is also accepted for list selectors other than urls). Use provider and capability IDs returned by discovery.' ) .option( '--request ', @@ -218,7 +298,7 @@ export function createFindToolsCommand(): Command { let call: Call = { provider: 'firecrawl', capability: 'find-tools', - options: parseToolOptions(options.options), + options: normalizeFindToolsOptions(parseToolOptions(options.options)), }; if (options.request) { if (urls.length || options.options) @@ -236,7 +316,7 @@ export function addAlexandriaScrapeOptions(command: Command): void { .addOption( new Option( '--alexandria ', - 'Execute a discovered tool through Scrape (repeat for batches)' + 'Execute a discovered tool through Scrape; also accepts a {"provider","capability","options"} JSON object (repeat for batches)' ).argParser((value: string, previous: string[] = []) => [ ...previous, value,