Skip to content
Merged

Fixes #277

Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "firecrawl-cli",
"version": "1.24.3",
"version": "1.24.4",
"publishConfig": {
"tag": "latest"
},
Expand Down
168 changes: 168 additions & 0 deletions src/__tests__/alexandria-beta.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -724,6 +724,174 @@ it('find-tools and explicit meta-tool execution use the same Scrape request', as
expect(requests[0].url).toBe('/v2/scrape');
});

it('accepts the HTTP contract object form for --alexandria and rejects malformed addresses locally', async () => {
const object = await cli([
'scrape',
'--alexandria',
'{"provider":"benzinga","capability":"news/search","options":{"pageSize":1}}',
'--request-id',
'object-1',
]);
expect(object.code).toBe(0);
const bare = await cli([
'scrape',
'--alexandria',
'benzinga/news/search',
'--options',
'{"pageSize":1}',
'--request-id',
'object-1',
]);
expect(bare.code).toBe(0);
expect(requests).toHaveLength(2);
expect(requests[0]).toEqual(requests[1]);
expect(requests[0].body.alexandria).toEqual([
{
provider: 'benzinga',
capability: 'news/search',
options: { pageSize: 1 },
},
]);
const separate = await cli([
'scrape',
'--alexandria',
'{"provider":"benzinga","capability":"news/search"}',
'--options',
'{"pageSize":2}',
]);
expect(separate.code).toBe(0);
expect(requests[2].body.alexandria[0].options).toEqual({ pageSize: 2 });
for (const address of [
'{"provider":"benzinga"}',
'{"provider":"benzinga","capability":"news/search","extra":1}',
'{not json',
'benzinga',
]) {
const result = await cli(['scrape', '--alexandria', address]);
expect(result.code).toBe(1);
expect(result.stderr).toContain('provider/capability');
}
const both = await cli([
'scrape',
'--alexandria',
'{"provider":"benzinga","capability":"news/search","options":{}}',
'--options',
'{}',
]);
expect(both.code).toBe(1);
expect(both.stderr).toContain('not both');
expect(requests).toHaveLength(3);
});

it('accepts the comma-separated expand form documented in find-tools --help', async () => {
const help = await cli(['find-tools', '--help']);
expect(help.stdout.replace(/\s+/g, ' ')).toContain(
'expand: ["options","response","examples"]'
);
expect(
(
await cli([
'find-tools',
'--options',
'{"providers":"fred","expand":"options, response,examples"}',
])
).code
).toBe(0);
expect(
(
await cli([
'find-tools',
'--options',
'{"providers":["fred"],"expand":["options","response","examples"]}',
])
).code
).toBe(0);
expect(
(
await cli([
'scrape',
'--alexandria',
'firecrawl/find-tools',
'--options',
'{"providers":"fred","expand":"options,response,examples"}',
])
).code
).toBe(0);
expect(requests).toHaveLength(3);
for (const request of requests)
expect(request.body.alexandria[0].options).toEqual({
providers: ['fred'],
expand: ['options', 'response', 'examples'],
});
});

it('never comma-splits urls, which may legally contain commas', async () => {
const url = 'https://example.com/items,23?ids=1,2';
expect(
(
await cli([
'find-tools',
'--options',
JSON.stringify({ urls: url, providers: 'fred,benzinga' }),
])
).code
).toBe(0);
expect(
(
await cli([
'find-tools',
'--request',
JSON.stringify({
provider: 'firecrawl',
capability: 'find-tools',
options: { urls: url, providers: 'fred,benzinga' },
}),
])
).code
).toBe(0);
expect(
(
await cli([
'scrape',
'--alexandria',
'firecrawl/find-tools',
'--options',
JSON.stringify({ urls: [url], providers: 'fred,benzinga' }),
])
).code
).toBe(0);
expect(requests).toHaveLength(3);
for (const request of requests)
expect(request.body.alexandria[0].options).toEqual({
urls: [url],
providers: ['fred', 'benzinga'],
});
});

it('trims whitespace around the slash in bare addresses and rejects empty parts', async () => {
const padded = await cli([
'scrape',
'--alexandria',
' benzinga / news/search ',
'--options',
'{"pageSize":1}',
]);
expect(padded.code).toBe(0);
expect(requests[0].body.alexandria).toEqual([
{
provider: 'benzinga',
capability: 'news/search',
options: { pageSize: 1 },
},
]);
for (const address of ['benzinga/ ', ' /news', '/', 'benzinga/']) {
const result = await cli(['scrape', '--alexandria', address]);
expect(result.code).toBe(1);
expect(result.stderr).toContain('provider/capability');
}
expect(requests).toHaveLength(1);
});

it('discovers a known URL and follows its returned meta-tool request without executing providers', async () => {
const next = {
provider: 'firecrawl',
Expand Down
100 changes: 90 additions & 10 deletions src/commands/alexandria.ts
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,78 @@ export function parseToolOptions(raw = '{}'): Record<string, unknown> {
return value;
}

const FIND_TOOLS_LIST_SELECTORS = [
'providers',
'categories',
'groups',
'capabilities',
'expand',
];

/**
* Find Tools list selectors take arrays; also accept the comma-separated form.
* `urls` is never comma-split (a comma is legal inside a URL); a single string is wrapped.
*/
export function normalizeFindToolsOptions(
options: Record<string, unknown>
): Record<string, unknown> {
const next = { ...options };
for (const key of FIND_TOOLS_LIST_SELECTORS) {
const value = next[key];
if (typeof value === 'string')
next[key] = value
.split(',')
.map((part) => part.trim())
.filter(Boolean);
}
if (typeof next.urls === 'string') next.urls = [next.urls.trim()];
return next;
}

const ADDRESS_HINT =
'Use a provider/capability address, or a JSON object {"provider":"...","capability":"...","options":{...}}.';

/** Accept both the bare `provider/capability` form and the HTTP contract's object form. */
function parseAddress(address: string): AlexandriaCall {
const trimmed = address.trim();
if (trimmed.startsWith('{')) {
let value: any;
try {
value = JSON.parse(trimmed);
} catch {
throw new Error(ADDRESS_HINT);
}
if (
!value ||
typeof value !== 'object' ||
Array.isArray(value) ||
typeof value.provider !== 'string' ||
!value.provider.trim() ||
typeof value.capability !== 'string' ||
!value.capability.trim() ||
(value.options !== undefined &&
(!value.options ||
typeof value.options !== 'object' ||
Array.isArray(value.options))) ||
Object.keys(value).some(
(key) => !['provider', 'capability', 'options'].includes(key)
)
)
throw new Error(ADDRESS_HINT);
return {
provider: value.provider.trim(),
capability: value.capability.trim(),
...(value.options !== undefined && { options: value.options }),
};
}
const slash = trimmed.indexOf('/');
if (slash < 0) throw new Error(ADDRESS_HINT);
const provider = trimmed.slice(0, slash).trim();
const capability = trimmed.slice(slash + 1).trim();
if (!provider || !capability) throw new Error(ADDRESS_HINT);
return { provider, capability };
}

export function buildCalls(addresses: string[], values: string[] = []): Call[] {
if (
!addresses.length ||
Expand All @@ -42,13 +114,19 @@ export function buildCalls(addresses: string[], values: string[] = []): Call[] {
'Provide 1-10 capabilities, with at most one --options value per capability.'
);
return addresses.map((address, i) => {
const slash = address.indexOf('/');
if (slash < 1 || slash === address.length - 1)
throw new Error('Use a provider/capability address.');
const call = parseAddress(address);
if (call.options !== undefined && values[i] !== undefined)
throw new Error(
`Provide options for ${call.provider}/${call.capability} either inside the object or with --options, not both.`
);
const options = call.options ?? parseToolOptions(values[i]);
return {
provider: address.slice(0, slash),
capability: address.slice(slash + 1),
options: parseToolOptions(values[i]),
provider: call.provider,
capability: call.capability,
options:
call.provider === 'firecrawl' && call.capability === 'find-tools'
? normalizeFindToolsOptions(options)
: options,
};
});
}
Expand Down Expand Up @@ -187,7 +265,9 @@ export function parseFindToolsRequest(raw: string): Call {
return {
provider: 'firecrawl',
capability: 'find-tools',
options: parseToolOptions(JSON.stringify(next.options)),
options: normalizeFindToolsOptions(
parseToolOptions(JSON.stringify(next.options))
),
};
}

Expand All @@ -199,7 +279,7 @@ export function createFindToolsCommand(): Command {
.argument('[urls...]', 'Known HTTP(S) URLs to find tools for')
.option(
'--options <json>',
'Discovery options: query, urls, providers, categories, groups, capabilities; level: providers|groups|tools; limit: 1-100; expand: options,response,examples. Use provider and capability IDs returned by discovery.'
'Discovery options: query, urls, providers, categories, groups, capabilities; level: providers|groups|tools; limit: 1-100; expand: ["options","response","examples"] (a comma-separated string is also accepted for list selectors other than urls). Use provider and capability IDs returned by discovery.'
)
.option(
'--request <json>',
Expand All @@ -218,7 +298,7 @@ export function createFindToolsCommand(): Command {
let call: Call = {
provider: 'firecrawl',
capability: 'find-tools',
options: parseToolOptions(options.options),
options: normalizeFindToolsOptions(parseToolOptions(options.options)),
Comment thread
cubic-dev-ai[bot] marked this conversation as resolved.
};
if (options.request) {
if (urls.length || options.options)
Expand All @@ -236,7 +316,7 @@ export function addAlexandriaScrapeOptions(command: Command): void {
.addOption(
new Option(
'--alexandria <provider/capability>',
'Execute a discovered tool through Scrape (repeat for batches)'
'Execute a discovered tool through Scrape; also accepts a {"provider","capability","options"} JSON object (repeat for batches)'
).argParser((value: string, previous: string[] = []) => [
...previous,
value,
Expand Down
Loading