Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "firecrawl-cli",
"version": "1.24.6",
"version": "1.25.0",
"publishConfig": {
"tag": "latest"
},
Expand Down
3 changes: 2 additions & 1 deletion src/__tests__/commands/parse.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,8 @@ describe('executeParse', () => {
];
expect(url).toBe('https://api.firecrawl.dev/v2/parse');
expect(init.method).toBe('POST');
expect(init.headers).toBeUndefined();
// No Authorization; X-Origin attributes the keyless call to the CLI.
expect(init.headers).toEqual({ 'X-Origin': 'cli' });

const options = JSON.parse(init.body.get('options') as string);
expect(options).toEqual({
Expand Down
81 changes: 47 additions & 34 deletions src/__tests__/utils/client.test.ts
Original file line number Diff line number Diff line change
@@ -1,66 +1,79 @@
/**
* Tests for keyless request errors
*
* The API links every keyless prompt to signup tagged `utm_medium=api`. The CLI
* must retag that link as `cli` so signups started from the CLI are attributed
* to it.
* The API links every keyless prompt to the caller's own opaque signup link,
* https://firecrawl.dev/k/<id>, which the site resolves to CLI attribution when
* the request came from the CLI. The CLI prints that link unchanged and tells
* the API it is the CLI with X-Origin, including on requests without a body.
*/

import { describe, it, expect, vi, afterEach } from 'vitest';
import {
keylessGet,
keylessRequest,
withCliSignupTag,
} from '../../utils/client';
import { keylessGet, keylessRequest } from '../../utils/client';

const API_LIMIT_MESSAGE = `You've hit Firecrawl's keyless free tier rate limit. To continue now, create a free API key at https://www.firecrawl.dev/signin?utm_source=keyless&utm_medium=api
const OWN_SIGNUP_URL = 'https://firecrawl.dev/k/7fq2xab9';

const API_LIMIT_MESSAGE = `You've hit Firecrawl's keyless free tier rate limit. To continue now, create a free API key at ${OWN_SIGNUP_URL}

Then authenticate with:
Authorization: Bearer YOUR_API_KEY`;

const CLI_SIGNUP_URL =
'https://www.firecrawl.dev/signin?utm_source=keyless&utm_medium=cli';
// Before the /k links, the API sent a UTM-tagged link. An API still sending it
// must not be rewritten into something else.
const LEGACY_LIMIT_MESSAGE =
"You've hit Firecrawl's keyless free tier rate limit. To continue now, create a free API key at https://www.firecrawl.dev/signin?utm_source=keyless&utm_medium=api";

function stubFetch(status: number, body: unknown) {
vi.stubGlobal(
'fetch',
vi.fn(async () => new Response(JSON.stringify(body), { status }))
const fetchMock = vi.fn(
async (_url: string, _init?: RequestInit) =>
new Response(JSON.stringify(body), { status })
);
vi.stubGlobal('fetch', fetchMock);
return fetchMock;
}

describe('withCliSignupTag', () => {
it('retags the keyless signup link as cli', () => {
const message = withCliSignupTag(API_LIMIT_MESSAGE);

expect(message).toContain(CLI_SIGNUP_URL);
expect(message).not.toContain('utm_medium=api');
});

it('leaves messages without the keyless signup link unchanged', () => {
expect(withCliSignupTag('Firecrawl request failed (HTTP 500)')).toBe(
'Firecrawl request failed (HTTP 500)'
);
});
});

describe('keyless requests', () => {
afterEach(() => {
vi.unstubAllGlobals();
});

it('reports the keyless limit with the cli signup link', async () => {
stubFetch(429, { success: false, error: API_LIMIT_MESSAGE });
it('reports the keyless limit with the API-issued signup link unchanged', async () => {
stubFetch(429, {
success: false,
error: API_LIMIT_MESSAGE,
signup_url: OWN_SIGNUP_URL,
});

await expect(
keylessRequest('/v2/scrape', { url: 'https://example.com' })
).rejects.toThrow(CLI_SIGNUP_URL);
).rejects.toThrow(API_LIMIT_MESSAGE);
});

it('reports the keyless limit on GET requests with the cli signup link', async () => {
it('reports the keyless limit on GET requests with the API-issued link', async () => {
stubFetch(429, { success: false, error: API_LIMIT_MESSAGE });

await expect(keylessGet('/v2/research/search?q=test')).rejects.toThrow(
CLI_SIGNUP_URL
OWN_SIGNUP_URL
);
});

it('no longer rewrites a legacy UTM link', async () => {
stubFetch(429, { success: false, error: LEGACY_LIMIT_MESSAGE });

await expect(
keylessRequest('/v2/scrape', { url: 'https://example.com' })
).rejects.toThrow(LEGACY_LIMIT_MESSAGE);
});

it('identifies the CLI with X-Origin on POST and GET requests', async () => {
const fetchMock = stubFetch(200, { success: true });

await keylessRequest('/v2/scrape', { url: 'https://example.com' });
await keylessGet('/v2/research/search?q=test');

for (const [, init] of fetchMock.mock.calls) {
const headers = init?.headers as Record<string, string>;
expect(headers['X-Origin']).toBe('cli');
expect(headers.Authorization).toBeUndefined();
}
});
});
15 changes: 6 additions & 9 deletions src/commands/interact.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
* Execute AI prompts or code against a scraped page in a live browser session
*/

import { getClient, isKeylessMode, withCliSignupTag } from '../utils/client';
import { getClient, isKeylessMode, KEYLESS_CLI_HEADERS } from '../utils/client';
import { getConfig, validateConfig } from '../utils/config';
import {
getScrapeId,
Expand Down Expand Up @@ -59,6 +59,7 @@ function buildHeaders(apiKey: string | undefined, keyless: boolean) {
if (!keyless && apiKey) {
headers.Authorization = `Bearer ${apiKey}`;
}
if (keyless) Object.assign(headers, KEYLESS_CLI_HEADERS);
return headers;
}

Expand Down Expand Up @@ -100,10 +101,8 @@ export async function handleInteractExecute(
if (!response.ok) {
const errorData = await response.json().catch(() => ({}));
throw new Error(
withCliSignupTag(
(errorData as any).error ||
`HTTP ${response.status}: ${response.statusText}`
)
(errorData as any).error ||
`HTTP ${response.status}: ${response.statusText}`
);
}

Expand Down Expand Up @@ -169,10 +168,8 @@ export async function handleInteractStop(
if (!response.ok) {
const errorData = await response.json().catch(() => ({}));
throw new Error(
withCliSignupTag(
(errorData as any).error ||
`HTTP ${response.status}: ${response.statusText}`
)
(errorData as any).error ||
`HTTP ${response.status}: ${response.statusText}`
);
}

Expand Down
15 changes: 10 additions & 5 deletions src/commands/parse.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import * as path from 'path';
import type { FormatOption } from 'firecrawl';
import type { ParseOptions, ParseResult } from '../types/parse';
import type { ScrapeFormat } from '../types/scrape';
import { getClient, isKeylessMode, withCliSignupTag } from '../utils/client';
import { getClient, isKeylessMode, KEYLESS_CLI_HEADERS } from '../utils/client';
import { getConfig, validateConfig } from '../utils/config';
import { handleScrapeOutput } from '../utils/output';

Expand Down Expand Up @@ -184,8 +184,14 @@ export async function executeParse(
try {
const response = await fetch(`${apiUrl}/v2/parse`, {
method: 'POST',
// Multipart options are parsed after auth, so the header carries the
// CLI origin to the keyless check.
headers:
!keyless && apiKey ? { Authorization: `Bearer ${apiKey}` } : undefined,
!keyless && apiKey
? { Authorization: `Bearer ${apiKey}` }
: keyless
? { ...KEYLESS_CLI_HEADERS }
: undefined,
body: form,
});

Expand All @@ -195,10 +201,9 @@ export async function executeParse(
const payload = (await response.json().catch(() => ({}))) as any;

if (!response.ok || payload?.success === false) {
const message = withCliSignupTag(
const message =
payload?.error ||
`HTTP ${response.status}: ${response.statusText || 'Request failed'}`
);
`HTTP ${response.status}: ${response.statusText || 'Request failed'}`;
return { success: false, error: message };
}

Expand Down
28 changes: 12 additions & 16 deletions src/utils/client.ts
Original file line number Diff line number Diff line change
Expand Up @@ -30,15 +30,15 @@ export function isKeylessMode(apiKey?: string, apiUrl?: string): boolean {
}

/**
* The API's keyless prompts link to signup tagged `utm_medium=api`. Retag them
* as `cli` so accounts created from the CLI are attributed to the CLI.
* Headers for keyless requests. The API reads X-Origin to attribute keyless
* use, and a keyless prompt's signup link, to the CLI; requests without a body
* (GET research and developer lookups, interact stop) carry nothing else.
* Keyless error messages are printed as the API sends them: their
* firecrawl.dev/k/<id> link already resolves to CLI attribution.
*/
export function withCliSignupTag(message: string): string {
return message.replaceAll(
'utm_source=keyless&utm_medium=api',
'utm_source=keyless&utm_medium=cli'
);
}
export const KEYLESS_CLI_HEADERS: Readonly<Record<string, string>> = {
'X-Origin': 'cli',
};

export async function keylessRequest(
path: string,
Expand All @@ -47,15 +47,13 @@ export async function keylessRequest(
const apiUrl = (getConfig().apiUrl || DEFAULT_API_URL).replace(/\/$/, '');
const response = await fetch(`${apiUrl}${path}`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
headers: { 'Content-Type': 'application/json', ...KEYLESS_CLI_HEADERS },
body: JSON.stringify(body),
});
const json: any = await response.json().catch(() => ({}));
if (!response.ok) {
throw new Error(
withCliSignupTag(
json?.error || `Firecrawl request failed (HTTP ${response.status})`
)
json?.error || `Firecrawl request failed (HTTP ${response.status})`
);
}
return json;
Expand All @@ -65,14 +63,12 @@ export async function keylessGet(path: string): Promise<any> {
const apiUrl = (getConfig().apiUrl || DEFAULT_API_URL).replace(/\/$/, '');
const response = await fetch(`${apiUrl}${path}`, {
method: 'GET',
headers: { 'Content-Type': 'application/json' },
headers: { 'Content-Type': 'application/json', ...KEYLESS_CLI_HEADERS },
});
const json: any = await response.json().catch(() => ({}));
if (!response.ok) {
throw new Error(
withCliSignupTag(
json?.error || `Firecrawl request failed (HTTP ${response.status})`
)
json?.error || `Firecrawl request failed (HTTP ${response.status})`
);
}
return json;
Expand Down
Loading