Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 39 additions & 7 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,24 @@ export FIRECRAWL_API_KEY=fc-your-api-key
firecrawl scrape https://example.com --api-key fc-your-api-key
```

### Self-hosted / Local Development

For self-hosted Firecrawl instances or local development, use the `--api-url` option:

```bash
# Use a local Firecrawl instance (no API key required)
firecrawl --api-url http://localhost:3002 scrape https://example.com

# Or set via environment variable
export FIRECRAWL_API_URL=http://localhost:3002
firecrawl scrape https://example.com

# Self-hosted with API key
firecrawl --api-url https://firecrawl.mycompany.com --api-key fc-xxx scrape https://example.com
```

When using a custom API URL (anything other than `https://api.firecrawl.dev`), authentication is automatically skipped, allowing you to use local instances without an API key.

---

## Commands
Expand Down Expand Up @@ -340,10 +358,15 @@ firecrawl credit-usage --json --pretty

---

### `config` - View configuration
### `config` - Configure and view settings

```bash
# View current configuration
firecrawl config

# Configure with custom API URL
firecrawl config --api-url https://firecrawl.mycompany.com
firecrawl config --api-url http://localhost:3002 --api-key fc-xxx
```

Shows authentication status and stored credentials location.
Expand All @@ -359,6 +382,10 @@ firecrawl login --method browser
firecrawl login --method manual
firecrawl login --api-key fc-xxx

# Login to self-hosted instance
firecrawl login --api-url https://firecrawl.mycompany.com
firecrawl login --api-url http://localhost:3002 --api-key fc-xxx

# Logout
firecrawl logout
```
Expand All @@ -369,12 +396,13 @@ firecrawl logout

These options work with any command:

| Option | Description |
| --------------------- | -------------------------------------------- |
| `--status` | Show version, auth, concurrency, and credits |
| `-k, --api-key <key>` | Use specific API key |
| `-V, --version` | Show version |
| `-h, --help` | Show help |
| Option | Description |
| --------------------- | ------------------------------------------------------ |
| `--status` | Show version, auth, concurrency, and credits |
| `-k, --api-key <key>` | Use specific API key |
| `--api-url <url>` | Use custom API URL (for self-hosted/local development) |
| `-V, --version` | Show version |
| `-h, --help` | Show help |

### Check Status

Expand Down Expand Up @@ -458,6 +486,10 @@ firecrawl https://example.com | grep -i "keyword"
# Set API key via environment
export FIRECRAWL_API_KEY=${{ secrets.FIRECRAWL_API_KEY }}
firecrawl crawl https://docs.example.com --wait -o docs.json

# Use self-hosted instance
export FIRECRAWL_API_URL=${{ secrets.FIRECRAWL_API_URL }}
firecrawl scrape https://example.com -o output.md
```

---
Expand Down
45 changes: 45 additions & 0 deletions src/__tests__/utils/config.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@ import {
getConfig,
resetConfig,
updateConfig,
validateConfig,
isCustomApiUrl,
} from '../../utils/config';
import { getClient, resetClient } from '../../utils/client';
import * as credentials from '../../utils/credentials';
Expand Down Expand Up @@ -231,4 +233,47 @@ describe('Config Fallback Priority', () => {
expect(config.apiUrl).toBe('https://url2.com'); // Should be updated
});
});

describe('isCustomApiUrl', () => {
it('should return false for default cloud API URL', () => {
initializeConfig({ apiUrl: 'https://api.firecrawl.dev' });
expect(isCustomApiUrl()).toBe(false);
});

it('should return true for custom API URLs', () => {
initializeConfig({ apiUrl: 'http://localhost:3002' });
expect(isCustomApiUrl()).toBe(true);
});

it('should return false when no apiUrl is set', () => {
initializeConfig({});
expect(isCustomApiUrl()).toBe(false);
});

it('should accept apiUrl parameter override', () => {
initializeConfig({ apiUrl: 'https://api.firecrawl.dev' });
expect(isCustomApiUrl('http://localhost:3002')).toBe(true);
});
});

describe('validateConfig with custom API URLs', () => {
it('should not require API key for custom API URLs', () => {
initializeConfig({ apiUrl: 'http://localhost:3002' });
// Should not throw
expect(() => validateConfig()).not.toThrow();
});

it('should require API key for cloud API URL', () => {
initializeConfig({ apiUrl: 'https://api.firecrawl.dev' });
expect(() => validateConfig()).toThrow('API key is required');
});

it('should not throw when API key is provided for cloud API', () => {
initializeConfig({
apiUrl: 'https://api.firecrawl.dev',
apiKey: 'fc-test-key',
});
expect(() => validateConfig()).not.toThrow();
});
});
});
4 changes: 2 additions & 2 deletions src/commands/crawl.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ async function checkCrawlStatus(
options: CrawlOptions
): Promise<CrawlStatusResult> {
try {
const app = getClient({ apiKey: options.apiKey });
const app = getClient({ apiKey: options.apiKey, apiUrl: options.apiUrl });
const status = await app.getCrawlStatus(jobId);

return {
Expand Down Expand Up @@ -48,7 +48,7 @@ export async function executeCrawl(
options: CrawlOptions
): Promise<CrawlResult | CrawlStatusResult> {
try {
const app = getClient({ apiKey: options.apiKey });
const app = getClient({ apiKey: options.apiKey, apiUrl: options.apiUrl });
const { urlOrJobId, status, wait, pollInterval, timeout } = options;

// If status flag is set or input looks like a job ID, check status
Expand Down
11 changes: 7 additions & 4 deletions src/commands/credit-usage.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@ export interface CreditUsageResult {
export interface CreditUsageOptions {
/** API key for Firecrawl */
apiKey?: string;
/** API URL for Firecrawl */
apiUrl?: string;
/** Output file path */
output?: string;
/** Output as JSON format */
Expand All @@ -36,17 +38,18 @@ export async function executeCreditUsage(
options: CreditUsageOptions = {}
): Promise<CreditUsageResult> {
try {
// Update config if API key provided (via getClient)
if (options.apiKey) {
getClient({ apiKey: options.apiKey });
// Update config if API key or URL provided (via getClient)
if (options.apiKey || options.apiUrl) {
getClient({ apiKey: options.apiKey, apiUrl: options.apiUrl });
}

// Get config and validate API key
const config = getConfig();
const apiKey = options.apiKey || config.apiKey;
validateConfig(apiKey);

const apiUrl = config.apiUrl || 'https://api.firecrawl.dev';
const apiUrl =
options.apiUrl || config.apiUrl || 'https://api.firecrawl.dev';

// Make the API call to /v2/team/credit-usage
const url = `${apiUrl.replace(/\/$/, '')}/v2/team/credit-usage`;
Expand Down
10 changes: 6 additions & 4 deletions src/commands/login.ts
Original file line number Diff line number Diff line change
Expand Up @@ -30,9 +30,10 @@ export async function handleLoginCommand(
): Promise<void> {
const apiUrl = options.apiUrl?.replace(/\/$/, '') || DEFAULT_API_URL;
const webUrl = options.webUrl?.replace(/\/$/, '') || WEB_URL;
const isCustomUrl = apiUrl !== DEFAULT_API_URL;

// If already authenticated, let them know
if (isAuthenticated() && !options.apiKey && !options.method) {
if (isAuthenticated() && !options.apiKey && !options.method && !isCustomUrl) {
console.log('You are already logged in.');
console.log(`Credentials stored at: ${getConfigDirectoryPath()}`);
console.log('\nTo login with a different account, run:');
Expand All @@ -43,7 +44,8 @@ export async function handleLoginCommand(

// If API key provided directly, save it
if (options.apiKey) {
if (!options.apiKey.startsWith('fc-')) {
// Only validate fc- prefix for cloud API
if (!isCustomUrl && !options.apiKey.startsWith('fc-')) {
console.error(
'Error: Invalid API key format. API keys should start with "fc-"'
);
Expand Down Expand Up @@ -75,11 +77,11 @@ export async function handleLoginCommand(
let result: { apiKey: string; apiUrl: string; teamName?: string };

if (options.method === 'manual') {
result = await manualLogin();
result = await manualLogin(apiUrl);
} else if (options.method === 'browser') {
result = await browserLogin(webUrl);
} else {
result = await interactiveLogin(webUrl);
result = await interactiveLogin(webUrl, apiUrl);
}

// Save credentials
Expand Down
2 changes: 1 addition & 1 deletion src/commands/map.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import { writeOutput } from '../utils/output';
*/
export async function executeMap(options: MapOptions): Promise<MapResult> {
try {
const app = getClient({ apiKey: options.apiKey });
const app = getClient({ apiKey: options.apiKey, apiUrl: options.apiUrl });
const { urlOrJobId } = options;

// Build map options
Expand Down
4 changes: 2 additions & 2 deletions src/commands/scrape.ts
Original file line number Diff line number Diff line change
Expand Up @@ -49,8 +49,8 @@ function outputTiming(
export async function executeScrape(
options: ScrapeOptions
): Promise<ScrapeResult> {
// Get client instance (updates global config if apiKey provided)
const app = getClient({ apiKey: options.apiKey });
// Get client instance (updates global config if apiKey/apiUrl provided)
const app = getClient({ apiKey: options.apiKey, apiUrl: options.apiUrl });

// Build scrape options
const formats: FormatOption[] = [];
Expand Down
2 changes: 1 addition & 1 deletion src/commands/search.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ export async function executeSearch(
options: SearchOptions
): Promise<SearchResult> {
try {
const app = getClient({ apiKey: options.apiKey });
const app = getClient({ apiKey: options.apiKey, apiUrl: options.apiUrl });

// Build search options for the SDK
const searchParams: Record<string, any> = {
Expand Down
25 changes: 22 additions & 3 deletions src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -48,20 +48,31 @@ program
'-k, --api-key <key>',
'Firecrawl API key (or set FIRECRAWL_API_KEY env var)'
)
.option('--api-url <url>', 'API URL (or set FIRECRAWL_API_URL env var)')
.option('--status', 'Show version, auth status, concurrency, and credits')
.allowUnknownOption() // Allow unknown options when URL is passed directly
.hook('preAction', async (thisCommand, actionCommand) => {
// Update global config if API key is provided via global option
// Update global config if API key or URL is provided via global option
const globalOptions = thisCommand.opts();
const commandOptions = actionCommand.opts();
if (globalOptions.apiKey) {
updateConfig({ apiKey: globalOptions.apiKey });
}
if (globalOptions.apiUrl) {
updateConfig({ apiUrl: globalOptions.apiUrl });
}

// Check if this command requires authentication
const commandName = actionCommand.name();
if (AUTH_REQUIRED_COMMANDS.includes(commandName)) {
// Ensure user is authenticated (prompts for login if needed)
await ensureAuthenticated();
// Skip auth for custom API URLs (e.g., local development)
// Check both global and command-level options
const { isCustomApiUrl } = await import('./utils/config');
const effectiveApiUrl = commandOptions.apiUrl || globalOptions.apiUrl;
if (!isCustomApiUrl(effectiveApiUrl)) {
// Ensure user is authenticated (prompts for login if needed)
await ensureAuthenticated();
}
}
});

Expand Down Expand Up @@ -98,6 +109,7 @@ function createScrapeCommand(): Command {
'-k, --api-key <key>',
'Firecrawl API key (overrides global --api-key)'
)
.option('--api-url <url>', 'API URL (overrides global --api-url)')
.option('-o, --output <path>', 'Output file path (default: stdout)')
.option('--json', 'Output as JSON format', false)
.option('--pretty', 'Pretty print JSON output', false)
Expand Down Expand Up @@ -199,6 +211,7 @@ function createCrawlCommand(): Command {
'-k, --api-key <key>',
'Firecrawl API key (overrides global --api-key)'
)
.option('--api-url <url>', 'API URL (overrides global --api-url)')
.option('-o, --output <path>', 'Output file path (default: stdout)')
.option('--pretty', 'Pretty print JSON output', false)
.action(async (positionalUrlOrJobId, options) => {
Expand All @@ -224,6 +237,7 @@ function createCrawlCommand(): Command {
output: options.output,
pretty: options.pretty,
apiKey: options.apiKey,
apiUrl: options.apiUrl,
limit: options.limit,
maxDepth: options.maxDepth,
excludePaths: options.excludePaths
Expand Down Expand Up @@ -273,6 +287,7 @@ function createMapCommand(): Command {
'-k, --api-key <key>',
'Firecrawl API key (overrides global --api-key)'
)
.option('--api-url <url>', 'API URL (overrides global --api-url)')
.option('-o, --output <path>', 'Output file path (default: stdout)')
.option('--json', 'Output as JSON format', false)
.option('--pretty', 'Pretty print JSON output', false)
Expand All @@ -293,6 +308,7 @@ function createMapCommand(): Command {
json: options.json,
pretty: options.pretty,
apiKey: options.apiKey,
apiUrl: options.apiUrl,
limit: options.limit,
search: options.search,
sitemap: options.sitemap,
Expand Down Expand Up @@ -363,6 +379,7 @@ function createSearchCommand(): Command {
'-k, --api-key <key>',
'Firecrawl API key (overrides global --api-key)'
)
.option('--api-url <url>', 'API URL (overrides global --api-url)')
.option('-o, --output <path>', 'Output file path (default: stdout)')
// .option(
// '-p, --pretty',
Expand Down Expand Up @@ -431,6 +448,7 @@ function createSearchCommand(): Command {
scrapeFormats,
onlyMainContent: options.onlyMainContent,
apiKey: options.apiKey,
apiUrl: options.apiUrl,
output: options.output,
json: options.json,
pretty: options.pretty,
Expand Down Expand Up @@ -520,6 +538,7 @@ program
'-k, --api-key <key>',
'Firecrawl API key (overrides global --api-key)'
)
.option('--api-url <url>', 'API URL (overrides global --api-url)')
.option('-o, --output <path>', 'Output file path (default: stdout)')
.option('--json', 'Output as JSON format', false)
.option(
Expand Down
2 changes: 2 additions & 0 deletions src/types/crawl.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@
export interface CrawlOptions {
/** API key for Firecrawl */
apiKey?: string;
/** API URL for Firecrawl */
apiUrl?: string;
/** URL to crawl or job ID to check status */
urlOrJobId: string;
/** Check status of existing crawl job */
Expand Down
2 changes: 2 additions & 0 deletions src/types/map.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@
export interface MapOptions {
/** API key for Firecrawl */
apiKey?: string;
/** API URL for Firecrawl */
apiUrl?: string;
/** URL to map or job ID to check status */
urlOrJobId: string;
/** Check status of existing map job */
Expand Down
2 changes: 2 additions & 0 deletions src/types/scrape.ts
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,8 @@ export interface ScrapeOptions {
excludeTags?: string[];
/** API key for Firecrawl */
apiKey?: string;
/** API URL for Firecrawl */
apiUrl?: string;
/** Output file path */
output?: string;
/** Pretty print JSON output */
Expand Down
2 changes: 2 additions & 0 deletions src/types/search.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@ export interface SearchOptions {
query: string;
/** API key for Firecrawl */
apiKey?: string;
/** API URL for Firecrawl */
apiUrl?: string;
/** Maximum number of results (default: 5, max: 100) */
limit?: number;
/** Sources to search: web, images, news (default: web) */
Expand Down
Loading