diff --git a/README.md b/README.md index 1b8a72716a..8303532e70 100644 --- a/README.md +++ b/README.md @@ -104,29 +104,37 @@ firecrawl https://example.com --format json -o data.json --pretty #### Scrape Options -| Option | Description | -| ------------------------ | ------------------------------------------------------- | -| `-f, --format ` | Output format(s), comma-separated | -| `-H, --html` | Shortcut for `--format html` | -| `--only-main-content` | Extract only main content (removes navs, footers, etc.) | -| `--wait-for ` | Wait time before scraping (for JS-rendered content) | -| `--screenshot` | Take a screenshot | -| `--include-tags ` | Only include specific HTML tags | -| `--exclude-tags ` | Exclude specific HTML tags | -| `-o, --output ` | Save output to file | -| `--pretty` | Pretty print JSON output | -| `--timing` | Show request timing info | +| Option | Description | +| -------------------------- | ------------------------------------------------------- | +| `-f, --format ` | Output format(s), comma-separated | +| `-H, --html` | Shortcut for `--format html` | +| `-S, --summary` | Shortcut for `--format summary` | +| `--only-main-content` | Extract only main content (removes navs, footers, etc.) | +| `--wait-for ` | Wait time before scraping (for JS-rendered content) | +| `--screenshot` | Take a screenshot | +| `--include-tags ` | Only include specific HTML tags | +| `--exclude-tags ` | Exclude specific HTML tags | +| `--max-age ` | Maximum age of cached content in milliseconds | +| `-o, --output ` | Save output to file | +| `--json` | Output as JSON format | +| `--pretty` | Pretty print JSON output | +| `--timing` | Show request timing info | #### Available Formats -| Format | Description | -| ------------ | -------------------------- | -| `markdown` | Clean markdown (default) | -| `html` | Cleaned HTML | -| `rawHtml` | Original HTML | -| `links` | All links on the page | -| `screenshot` | Screenshot as base64 | -| `json` | Structured JSON extraction | +| Format | Description | +| ---------------- | ---------------------------- | +| `markdown` | Clean markdown (default) | +| `html` | Cleaned HTML | +| `rawHtml` | Original HTML | +| `links` | All links on the page | +| `images` | All images on the page | +| `screenshot` | Screenshot as base64 | +| `summary` | AI-generated summary | +| `json` | Structured JSON extraction | +| `changeTracking` | Track changes on the page | +| `attributes` | Page attributes and metadata | +| `branding` | Brand identity extraction | #### Examples @@ -209,7 +217,7 @@ firecrawl search "web scraping" | `--scrape-formats ` | Scrape formats when `--scrape` enabled (default: markdown) | | `--only-main-content` | Include only main content when scraping (default: true) | | `-o, --output ` | Save to file | -| `--json` | Output as compact JSON (use `-p` for pretty JSON) | +| `--json` | Output as compact JSON | #### Examples @@ -221,10 +229,10 @@ firecrawl search "React Server Components" --tbs qdr:m --limit 10 firecrawl search "web scraping library" --categories github --limit 20 # Search and get full content -firecrawl search "firecrawl documentation" --scrape --scrape-formats markdown -p -o results.json +firecrawl search "firecrawl documentation" --scrape --scrape-formats markdown --json -o results.json # Find research papers -firecrawl search "large language models" --categories research -p +firecrawl search "large language models" --categories research --json # Search with location targeting firecrawl search "best coffee shops" --location "Berlin,Germany" --country DE @@ -426,17 +434,97 @@ firecrawl agent abc123-def456-... --wait --poll-interval 10 --- -### `config` - Configure and view settings +### `browser` - Cloud browser sessions (Beta) + +Launch and control cloud browser sessions. By default, commands are sent to agent-browser (pre-installed in every sandbox). Use `--python` or `--node` to run Playwright code directly instead. + +```bash +# 1. Launch a session +firecrawl browser launch --stream + +# 2. Execute agent-browser commands (default) +firecrawl browser execute "open https://example.com" +firecrawl browser execute "snapshot" +firecrawl browser execute "click @e5" +firecrawl browser execute "scrape" + +# 3. Execute Playwright Python or JavaScript +firecrawl browser execute --python "await page.goto('https://example.com'); print(await page.title())" +firecrawl browser execute --node "await page.goto('https://example.com'); await page.title()" + +# 4. List sessions +firecrawl browser list + +# 5. Close +firecrawl browser close +``` + +#### Launch Options + +| Option | Description | +| ---------------------------- | ------------------------------------------- | +| `--ttl ` | Total session TTL in seconds (default: 300) | +| `--ttl-inactivity ` | Inactivity TTL in seconds | +| `--stream` | Enable live view streaming | +| `-o, --output ` | Save output to file | +| `--json` | Output as JSON format | + +#### Execute Options + +| Option | Description | +| --------------------- | ------------------------------------------------------------------ | +| `--python` | Execute as Playwright Python code | +| `--node` | Execute as Playwright JavaScript code | +| `--bash` | Execute bash commands in the sandbox (agent-browser pre-installed) | +| `--session ` | Target a specific session (auto-saved on launch) | +| `-o, --output ` | Save output to file | +| `--json` | Output as JSON format | + +By default (no flag), commands are sent to agent-browser. `--python`, `--node`, and `--bash` are mutually exclusive. + +#### Examples ```bash -# View current configuration -firecrawl config +# agent-browser commands (default mode) +firecrawl browser execute "open https://example.com" +firecrawl browser execute "snapshot" +firecrawl browser execute "click @e5" +firecrawl browser execute "fill @e3 'search query'" +firecrawl browser execute "scrape" +# Playwright Python +firecrawl browser execute --python "await page.goto('https://example.com'); print(await page.title())" + +# Playwright JavaScript +firecrawl browser execute --node "await page.goto('https://example.com'); await page.title()" + +# Bash (arbitrary commands in the sandbox) +firecrawl browser execute --bash "ls /tmp" + +# Launch with extended TTL +firecrawl browser launch --ttl 900 --ttl-inactivity 120 + +# JSON output +firecrawl browser execute --json "snapshot" +``` + +--- + +### `config` - Configure settings + +```bash # Configure with custom API URL firecrawl config --api-url https://firecrawl.mycompany.com firecrawl config --api-url http://localhost:3002 --api-key fc-xxx ``` +### `view-config` - View current configuration + +```bash +# View current configuration and authentication status +firecrawl view-config +``` + Shows authentication status and stored credentials location. --- @@ -479,7 +567,7 @@ firecrawl --status ``` ``` - 🔥 firecrawl cli v1.0.2 + 🔥 firecrawl cli v1.4.0 ● Authenticated via stored credentials Concurrency: 0/100 jobs (parallel scrape limit) diff --git a/package.json b/package.json index ef524813ba..62700653be 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "firecrawl-cli", - "version": "1.3.1", + "version": "1.4.0", "description": "Command-line interface for Firecrawl. Scrape, crawl, and extract data from any website directly from your terminal.", "main": "dist/index.js", "bin": { @@ -64,7 +64,7 @@ "vitest": "^4.0.0" }, "dependencies": { - "@mendable/firecrawl-js": "^4.12.0", + "@mendable/firecrawl-js": "4.13.0", "commander": "^14.0.2" } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f61e2fd09f..9395731996 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -9,8 +9,8 @@ importers: .: dependencies: '@mendable/firecrawl-js': - specifier: ^4.12.0 - version: 4.12.0 + specifier: 4.13.0 + version: 4.13.0 commander: specifier: ^14.0.2 version: 14.0.2 @@ -195,8 +195,8 @@ packages: '@jridgewell/sourcemap-codec@1.5.5': resolution: {integrity: sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==} - '@mendable/firecrawl-js@4.12.0': - resolution: {integrity: sha512-Jjawuumet+3htp39PwwhkZhSj2ORR8Tz/HqORoFGngrB3HadMBKoX6SSPIhayRTXwNnDkaU0PWm1SUtRvPdoPw==} + '@mendable/firecrawl-js@4.13.0': + resolution: {integrity: sha512-8Y2+jjol4/ZXjxyf/5VUGCeOeGYWOOp03HZoyULWU5/adyUjnNL1DkI0/96wHs1aGL5/+JCq/hDfkXMUFBVYfQ==} engines: {node: '>=22.0.0'} '@rollup/rollup-android-arm-eabi@4.55.1': @@ -387,8 +387,8 @@ packages: asynckit@0.4.0: resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==} - axios@1.13.2: - resolution: {integrity: sha512-VPk9ebNqPcy5lRGuSlKx752IlDatOjT9paPlm8A7yOuW2Fbvp4X3JznJtT4f0GzGLLiWE9W8onz51SqLYwzGaA==} + axios@1.13.5: + resolution: {integrity: sha512-cz4ur7Vb0xS4/KUN0tPWe44eqxrIu31me+fbang3ijiNscE129POzipJJA6zniq2C/Z6sJCjMimjS8Lc/GAs8Q==} braces@3.0.3: resolution: {integrity: sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==} @@ -972,9 +972,9 @@ snapshots: '@jridgewell/sourcemap-codec@1.5.5': {} - '@mendable/firecrawl-js@4.12.0': + '@mendable/firecrawl-js@4.13.0': dependencies: - axios: 1.13.2 + axios: 1.13.5 typescript-event-target: 1.1.2 zod: 3.25.76 zod-to-json-schema: 3.25.1(zod@3.25.76) @@ -1122,7 +1122,7 @@ snapshots: asynckit@0.4.0: {} - axios@1.13.2: + axios@1.13.5: dependencies: follow-redirects: 1.15.11 form-data: 4.0.5 diff --git a/skills/firecrawl-cli/SKILL.md b/skills/firecrawl-cli/SKILL.md index 46eba9e844..5c6eb6cb83 100644 --- a/skills/firecrawl-cli/SKILL.md +++ b/skills/firecrawl-cli/SKILL.md @@ -32,7 +32,7 @@ firecrawl --status Output when ready: ``` - 🔥 firecrawl cli v1.0.2 + 🔥 firecrawl cli v1.4.0 ● Authenticated via FIRECRAWL_API_KEY Concurrency: 0/100 jobs (parallel scrape limit) @@ -210,6 +210,214 @@ firecrawl map https://example.com --include-subdomains -o .firecrawl/all-urls.tx - `--json` - Output as JSON - `-o, --output ` - Save to file +### Browser - Cloud browser sessions + +Launch remote Chromium sessions with a built-in sandbox. By default, commands are sent to agent-browser (pre-installed in every sandbox). Use `--python` or `--node` to run Playwright code directly instead. Sessions persist across commands -- after `launch`, subsequent `execute` calls reuse the last session automatically. + +```bash +# Launch a session (saves connection details) +firecrawl browser launch -o .firecrawl/browser-session.json --json + +# Launch with custom TTL and live view streaming +firecrawl browser launch --ttl 600 --stream -o .firecrawl/browser-session.json --json + +# Execute agent-browser commands (default mode) +firecrawl browser execute "open https://example.com" -o .firecrawl/browser-result.txt +firecrawl browser execute "snapshot" -o .firecrawl/browser-result.txt +firecrawl browser execute "click @e5" +firecrawl browser execute "scrape" -o .firecrawl/browser-scrape.md + +# Execute Playwright Python +firecrawl browser execute --python 'await page.goto("https://example.com") +print(await page.title())' -o .firecrawl/browser-result.txt + +# Execute Playwright JavaScript +firecrawl browser execute --node 'await page.goto("https://example.com"); await page.title()' -o .firecrawl/browser-result.txt + +# Execute arbitrary bash in the sandbox +firecrawl browser execute --bash 'ls /tmp' -o .firecrawl/browser-result.txt + +# Execute against a specific session +firecrawl browser execute --session "snapshot" -o .firecrawl/browser-result.txt + +# List all sessions +firecrawl browser list --json -o .firecrawl/browser-sessions.json + +# List only active sessions +firecrawl browser list active --json -o .firecrawl/browser-sessions.json + +# Close last session +firecrawl browser close + +# Close a specific session +firecrawl browser close --session +``` + +**Browser Options:** + +- `--ttl ` - Total session lifetime (default: 300) +- `--ttl-inactivity ` - Auto-close after inactivity +- `--stream` - Enable live view streaming +- `--python` - Execute as Playwright Python code +- `--node` - Execute as Playwright JavaScript code +- `--bash` - Execute bash commands in the sandbox (agent-browser pre-installed, CDP_URL auto-injected) +- `--session ` - Target specific session (default: last launched session) +- `-o, --output ` - Save to file + +By default (no flag), commands are sent to agent-browser. `--python`, `--node`, and `--bash` are mutually exclusive. + +**Notes:** + +- Session auto-saves after `launch` -- no need to pass `--session` for subsequent commands +- In Python/Node mode, `page`, `browser`, and `context` objects are pre-configured (no setup needed) +- Use `print()` to return output from Python execution + +### Crawl - Crawl an entire website + +```bash +# Start a crawl (returns job ID) +firecrawl crawl https://example.com -o .firecrawl/crawl-result.json + +# Wait for crawl to complete +firecrawl crawl https://example.com --wait -o .firecrawl/crawl-result.json --pretty + +# With progress indicator +firecrawl crawl https://example.com --wait --progress -o .firecrawl/crawl-result.json + +# Check crawl status +firecrawl crawl + +# Limit pages and depth +firecrawl crawl https://example.com --limit 100 --max-depth 3 --wait -o .firecrawl/crawl-result.json + +# Crawl specific sections only +firecrawl crawl https://example.com --include-paths /blog,/docs --wait -o .firecrawl/crawl-blog.json + +# Exclude pages +firecrawl crawl https://example.com --exclude-paths /admin,/login --wait -o .firecrawl/crawl-result.json + +# Rate-limited crawl +firecrawl crawl https://example.com --delay 1000 --max-concurrency 2 --wait -o .firecrawl/crawl-result.json +``` + +**Crawl Options:** + +- `--wait` - Wait for crawl to complete before returning results +- `--progress` - Show progress while waiting +- `--limit ` - Maximum pages to crawl +- `--max-depth ` - Maximum crawl depth +- `--include-paths ` - Only crawl matching paths (comma-separated) +- `--exclude-paths ` - Skip matching paths (comma-separated) +- `--sitemap ` - include, skip, or only +- `--allow-subdomains` - Include subdomains +- `--allow-external-links` - Follow external links +- `--crawl-entire-domain` - Crawl entire domain +- `--ignore-query-parameters` - Treat URLs with different params as same +- `--delay ` - Delay between requests +- `--max-concurrency ` - Max concurrent requests +- `--poll-interval ` - Status check interval when waiting +- `--timeout ` - Timeout when waiting +- `-o, --output ` - Save to file +- `--pretty` - Pretty print JSON output + +### Agent - AI-powered web data extraction + +Run an AI agent that autonomously browses and extracts structured data from the web. Agent tasks typically take 2 to 5 minutes. + +```bash +# Basic usage (returns job ID immediately) +firecrawl agent "Find the pricing plans for Firecrawl" -o .firecrawl/agent-pricing.json + +# Wait for completion +firecrawl agent "Extract all product names and prices" --wait -o .firecrawl/agent-products.json + +# Focus on specific URLs +firecrawl agent "Get the main features listed" --urls https://example.com/features --wait -o .firecrawl/agent-features.json + +# Use structured output with JSON schema +firecrawl agent "Extract company info" --schema '{"type":"object","properties":{"name":{"type":"string"},"employees":{"type":"number"}}}' --wait -o .firecrawl/agent-company.json + +# Load schema from file +firecrawl agent "Extract product data" --schema-file ./product-schema.json --wait -o .firecrawl/agent-products.json + +# Use higher accuracy model +firecrawl agent "Extract detailed specs" --model spark-1-pro --wait -o .firecrawl/agent-specs.json + +# Limit cost +firecrawl agent "Get all blog post titles" --urls https://blog.example.com --max-credits 100 --wait -o .firecrawl/agent-blog.json + +# Check status of an existing job +firecrawl agent +firecrawl agent --wait +``` + +**Agent Options:** + +- `--urls ` - Comma-separated URLs to focus extraction on +- `--model ` - spark-1-mini (default, cheaper) or spark-1-pro (higher accuracy) +- `--schema ` - JSON schema for structured output (inline JSON string) +- `--schema-file ` - Path to JSON schema file +- `--max-credits ` - Maximum credits to spend (job fails if exceeded) +- `--wait` - Wait for agent to complete +- `--poll-interval ` - Polling interval when waiting (default: 5) +- `--timeout ` - Timeout when waiting +- `-o, --output ` - Save to file +- `--json` - Output as JSON format +- `--pretty` - Pretty print JSON output + +### Credit Usage - Check your credits + +```bash +# Show credit usage (human-readable) +firecrawl credit-usage + +# Output as JSON +firecrawl credit-usage --json --pretty -o .firecrawl/credits.json +``` + +### Bash Mode with agent-browser (Recommended for AI Agents) + +For AI agents, `--bash` with agent-browser is the simplest approach. agent-browser is pre-installed in every sandbox with 40+ commands. + +```bash +# Launch a session +firecrawl browser launch -o .firecrawl/browser-session.json --json + +# Navigate to a page +firecrawl browser execute --bash "agent-browser open https://example.com" + +# Snapshot: get an accessibility tree with @ref IDs +firecrawl browser execute --bash "agent-browser snapshot" + +# Interact using @ref IDs from the snapshot +firecrawl browser execute --bash "agent-browser fill @e3 'search query'" +firecrawl browser execute --bash "agent-browser click @e8" + +# Snapshot again to see updated state +firecrawl browser execute --bash "agent-browser snapshot" + +# Extract page content +firecrawl browser execute --bash "agent-browser scrape" -o .firecrawl/browser-scrape.md + +# Close +firecrawl browser close +``` + +**Core agent-browser commands:** + +| Command | Description | +| -------------------- | -------------------------------------- | +| `open ` | Navigate to a URL | +| `snapshot` | Get accessibility tree with `@ref` IDs | +| `screenshot` | Capture a PNG screenshot | +| `click <@ref>` | Click an element by ref | +| `type <@ref> ` | Type into an element | +| `fill <@ref> ` | Fill a form field (clears first) | +| `scrape` | Extract page content as markdown | +| `scroll ` | Scroll up/down/left/right | +| `wait ` | Wait for a duration | +| `eval ` | Evaluate JavaScript on the page | + ## Reading Scraped Files NEVER read entire firecrawl output files at once unless explicitly asked or required - they're often 1000+ lines. Instead, use grep, head, or incremental reads. Determine values dynamically based on file size and what you're looking for. diff --git a/src/__tests__/commands/browser.test.ts b/src/__tests__/commands/browser.test.ts new file mode 100644 index 0000000000..881dea1ba8 --- /dev/null +++ b/src/__tests__/commands/browser.test.ts @@ -0,0 +1,166 @@ +/** + * Browser commands – minimal viable tests + */ + +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import { + handleBrowserLaunch, + handleBrowserExecute, + handleBrowserList, + handleBrowserClose, + handleBrowserQuickExecute, +} from '../../commands/browser'; +import { getClient } from '../../utils/client'; +import { initializeConfig } from '../../utils/config'; +import { setupTest, teardownTest } from '../utils/mock-client'; + +vi.mock('child_process', () => ({ spawn: vi.fn() })); + +vi.mock('../../utils/client', async () => { + const actual = await vi.importActual('../../utils/client'); + return { ...actual, getClient: vi.fn() }; +}); + +vi.mock('../../utils/browser-session', async () => { + const actual = await vi.importActual('../../utils/browser-session'); + return { + ...actual, + saveBrowserSession: vi.fn(), + loadBrowserSession: vi.fn().mockReturnValue({ + id: 'stored-session-id', + cdpUrl: 'wss://stored', + createdAt: '2025-01-01T00:00:00Z', + }), + clearBrowserSession: vi.fn(), + getSessionId: vi.fn((override?: string) => override || 'stored-session-id'), + }; +}); + +vi.mock('../../utils/output', () => ({ writeOutput: vi.fn() })); + +const mockExit = vi + .spyOn(process, 'exit') + .mockImplementation((() => {}) as any); +const mockConsoleError = vi + .spyOn(console, 'error') + .mockImplementation(() => {}); +const mockConsoleLog = vi.spyOn(console, 'log').mockImplementation(() => {}); + +describe('Browser Commands', () => { + let mockClient: any; + + beforeEach(() => { + setupTest(); + initializeConfig({ + apiKey: 'test-api-key', + apiUrl: 'https://api.firecrawl.dev', + }); + mockClient = { + browser: vi.fn(), + browserExecute: vi.fn(), + listBrowsers: vi.fn(), + deleteBrowser: vi.fn(), + }; + vi.mocked(getClient).mockReturnValue(mockClient as any); + }); + + afterEach(() => { + teardownTest(); + vi.clearAllMocks(); + }); + + it('launch saves session on success', async () => { + mockClient.browser.mockResolvedValue({ + success: true, + id: 'session-123', + cdpUrl: 'wss://cdp.example.com/session-123', + liveViewUrl: 'https://live.example.com/browser-id', + }); + + await handleBrowserLaunch({}); + + const { saveBrowserSession } = await import('../../utils/browser-session'); + expect(saveBrowserSession).toHaveBeenCalledWith( + expect.objectContaining({ id: 'session-123' }) + ); + }); + + it('launch exits 1 on failure', async () => { + mockClient.browser.mockResolvedValue({ + success: false, + error: 'Not authorized', + }); + + await handleBrowserLaunch({}); + + expect(mockExit).toHaveBeenCalledWith(1); + }); + + it('execute sends python code to correct session', async () => { + mockClient.browserExecute.mockResolvedValue({ + success: true, + result: 'Example Domain', + }); + + await handleBrowserExecute({ code: 'await page.title()' }); + + expect(mockClient.browserExecute).toHaveBeenCalledWith( + 'stored-session-id', + { + code: 'await page.title()', + language: 'python', + } + ); + }); + + it('list returns sessions', async () => { + mockClient.listBrowsers.mockResolvedValue({ success: true, sessions: [] }); + + await handleBrowserList({}); + + expect(mockClient.listBrowsers).toHaveBeenCalledTimes(1); + }); + + it('close deletes and clears stored session', async () => { + mockClient.deleteBrowser.mockResolvedValue({ success: true }); + + await handleBrowserClose({}); + + expect(mockClient.deleteBrowser).toHaveBeenCalledWith('stored-session-id'); + const { clearBrowserSession } = await import('../../utils/browser-session'); + expect(clearBrowserSession).toHaveBeenCalled(); + }); + + it('quick execute skips launch when session exists', async () => { + // loadBrowserSession already returns a stored session (from mock) + // so quick execute should NOT call browser() to launch + await handleBrowserQuickExecute({ code: 'open https://example.com' }); + + expect(mockClient.browser).not.toHaveBeenCalled(); + }); + + it('quick execute auto-launches when no session exists', async () => { + const { loadBrowserSession, saveBrowserSession } = + await import('../../utils/browser-session'); + vi.mocked(loadBrowserSession).mockReturnValueOnce(null); // no session first call + vi.mocked(loadBrowserSession).mockReturnValue({ + // session exists after launch + id: 'new-session', + cdpUrl: 'wss://new', + createdAt: '2025-01-01T00:00:00Z', + }); + + mockClient.browser.mockResolvedValue({ + success: true, + id: 'new-session', + cdpUrl: 'wss://new', + }); + + await handleBrowserQuickExecute({ code: 'open https://example.com' }); + + expect(mockClient.browser).toHaveBeenCalledTimes(1); + expect(saveBrowserSession).toHaveBeenCalledWith( + expect.objectContaining({ id: 'new-session' }) + ); + }); +}); diff --git a/src/commands/browser.ts b/src/commands/browser.ts new file mode 100644 index 0000000000..4cbb76c439 --- /dev/null +++ b/src/commands/browser.ts @@ -0,0 +1,444 @@ +/** + * Browser command implementation + * Manages cloud browser sessions via the Firecrawl SDK + */ + +import { spawn } from 'child_process'; +import { getClient } from '../utils/client'; +import { + saveBrowserSession, + loadBrowserSession, + clearBrowserSession, + getSessionId, +} from '../utils/browser-session'; +import { writeOutput } from '../utils/output'; + +export interface BrowserLaunchOptions { + ttl?: number; + ttlInactivity?: number; + stream?: boolean; + apiKey?: string; + apiUrl?: string; + output?: string; + json?: boolean; +} + +export interface BrowserExecuteOptions { + code: string; + language?: 'python' | 'node' | 'bash'; + session?: string; + apiKey?: string; + apiUrl?: string; + output?: string; + json?: boolean; +} + +export interface BrowserListOptions { + status?: 'active' | 'destroyed'; + apiKey?: string; + apiUrl?: string; + output?: string; + json?: boolean; +} + +export interface BrowserCloseOptions { + session?: string; + apiKey?: string; + apiUrl?: string; + output?: string; + json?: boolean; +} + +export interface BrowserQuickExecuteOptions { + code: string; + apiKey?: string; + apiUrl?: string; + output?: string; + json?: boolean; +} + +/** + * Launch a new browser session + */ +export async function handleBrowserLaunch( + options: BrowserLaunchOptions +): Promise { + try { + const app = getClient({ apiKey: options.apiKey, apiUrl: options.apiUrl }); + + const args: { + ttl?: number; + activityTtl?: number; + streamWebView?: boolean; + } = {}; + if (options.ttl !== undefined) args.ttl = options.ttl; + if (options.ttlInactivity !== undefined) + args.activityTtl = options.ttlInactivity; + if (options.stream !== undefined) args.streamWebView = options.stream; + + const data = await app.browser(args); + + if (!data.success) { + console.error('Error:', data.error || 'Unknown error'); + process.exit(1); + } + + // Save session for future commands + saveBrowserSession({ + id: data.id!, + cdpUrl: data.cdpUrl!, + createdAt: new Date().toISOString(), + }); + + if (options.json) { + const output = JSON.stringify(data, null, 2); + writeOutput(output, options.output, !!options.output); + } else { + const lines: string[] = []; + lines.push(`Session ID: ${data.id}`); + lines.push(`CDP URL: ${data.cdpUrl}`); + if (data.liveViewUrl) { + lines.push(`Live View URL: ${data.liveViewUrl}`); + } + writeOutput(lines.join('\n'), options.output, !!options.output); + } + } catch (error) { + console.error( + 'Error:', + error instanceof Error ? error.message : 'Unknown error occurred' + ); + process.exit(1); + } +} + +/** + * Execute a bash command locally with CDP_URL and SESSION_ID env vars. + * Auto-injects --cdp for agent-browser commands. + */ +export function executeBashLocally( + command: string, + session: { id: string; cdpUrl: string } +): Promise<{ stdout: string; stderr: string; exitCode: number }> { + return new Promise((resolve) => { + // Auto-inject --cdp for agent-browser commands + let finalCommand = command; + if (command.startsWith('agent-browser') && !command.includes('--cdp')) { + const parts = command.split(' '); + // Insert --cdp after "agent-browser " + const insertIdx = Math.min(2, parts.length); + parts.splice(insertIdx, 0, '--cdp', `'${session.cdpUrl}'`); + finalCommand = parts.join(' '); + } + + const child = spawn('sh', ['-c', finalCommand], { + env: { + ...process.env, + CDP_URL: session.cdpUrl, + SESSION_ID: session.id, + }, + stdio: ['inherit', 'pipe', 'pipe'], + }); + + let stdout = ''; + let stderr = ''; + + child.stdout.on('data', (data: Buffer) => { + stdout += data.toString(); + }); + child.stderr.on('data', (data: Buffer) => { + stderr += data.toString(); + }); + + child.on('close', (code: number | null) => { + resolve({ stdout, stderr, exitCode: code ?? 1 }); + }); + }); +} + +/** + * Detect if an error indicates an expired or destroyed browser session. + */ +function isSessionExpiredError(error: unknown): boolean { + const msg = + error instanceof Error + ? error.message.toLowerCase() + : String(error).toLowerCase(); + const status = (error as { status?: number }).status; + if (status === 410 || status === 404) return true; + return /destroyed|expired|not found|gone|session.*closed/i.test(msg); +} + +/** + * Execute code in a browser session + */ +export async function handleBrowserExecute( + options: BrowserExecuteOptions +): Promise { + // Bash execution runs locally — skip the API entirely + if (options.language === 'bash') { + try { + let session: { id: string; cdpUrl: string } | null = null; + + if (options.session) { + const stored = loadBrowserSession(); + if (stored && stored.id === options.session) { + session = stored; + } else { + // Fetch CDP URL from the API for the specified session + const app = getClient({ + apiKey: options.apiKey, + apiUrl: options.apiUrl, + }); + const list = await app.listBrowsers({ status: 'active' }); + const match = list.sessions?.find( + (s: { id: string }) => s.id === options.session + ); + if (match && match.cdpUrl) { + session = { id: match.id, cdpUrl: match.cdpUrl }; + } else { + console.error( + `Error: Session ${options.session} not found or not active.` + ); + process.exit(1); + return; + } + } + } else { + session = loadBrowserSession(); + } + + if (!session) { + console.error( + 'Error: No active browser session. Run `firecrawl browser launch` first.' + ); + process.exit(1); + return; + } + + const result = await executeBashLocally(options.code, session); + + if (result.exitCode !== 0) { + if (result.stderr) process.stderr.write(result.stderr); + if (result.stdout) process.stdout.write(result.stdout); + process.exit(1); + } + + if (options.json) { + const data = { + success: true, + result: result.stdout.trimEnd(), + exitCode: result.exitCode, + }; + const output = JSON.stringify(data, null, 2); + writeOutput(output, options.output, !!options.output); + } else { + if (result.stdout) { + writeOutput( + result.stdout.trimEnd(), + options.output, + !!options.output + ); + } + } + } catch (error) { + console.error( + 'Error:', + error instanceof Error ? error.message : 'Unknown error occurred' + ); + process.exit(1); + } + + return; + } + + try { + const sessionId = getSessionId(options.session); + const app = getClient({ apiKey: options.apiKey, apiUrl: options.apiUrl }); + + const data = await app.browserExecute(sessionId, { + code: options.code, + language: options.language || 'python', + }); + + if (!data.success) { + console.error('Error:', data.error || 'Unknown error'); + process.exit(1); + } + + if (data.error) { + // Execution succeeded but code had an error + process.stderr.write(`Code error: ${data.error}\n`); + } + + if (options.json) { + const output = JSON.stringify(data, null, 2); + writeOutput(output, options.output, !!options.output); + } else { + if (data.result !== undefined && data.result !== '') { + writeOutput(data.result, options.output, !!options.output); + } + } + } catch (error) { + if (isSessionExpiredError(error)) { + const sessionId = + options.session || loadBrowserSession()?.id || 'unknown'; + console.error( + `Error: Session ${sessionId} has expired or been destroyed.\n` + + 'The session may have exceeded its TTL or been closed.\n' + + 'Start a new session with: firecrawl browser launch' + ); + // Clear stale stored session + const stored = loadBrowserSession(); + if (stored && !options.session) { + clearBrowserSession(); + } + } else { + console.error( + 'Error:', + error instanceof Error ? error.message : 'Unknown error occurred' + ); + } + process.exit(1); + } +} + +/** + * List browser sessions + */ +export async function handleBrowserList( + options: BrowserListOptions +): Promise { + try { + const app = getClient({ apiKey: options.apiKey, apiUrl: options.apiUrl }); + + const data = await app.listBrowsers( + options.status ? { status: options.status } : undefined + ); + + if (!data.success) { + console.error('Error:', data.error || 'Unknown error'); + process.exit(1); + } + + const sessions = data.sessions || []; + + if (options.json) { + const output = JSON.stringify(data, null, 2); + writeOutput(output, options.output, !!options.output); + } else { + if (sessions.length === 0) { + writeOutput( + 'No active browser sessions.', + options.output, + !!options.output + ); + } else { + const lines: string[] = []; + for (const s of sessions) { + const age = s.createdAt + ? `created ${new Date(s.createdAt).toLocaleString()}` + : ''; + lines.push(`${s.id} ${s.status} ${age}`); + } + writeOutput(lines.join('\n'), options.output, !!options.output); + } + } + } catch (error) { + console.error( + 'Error:', + error instanceof Error ? error.message : 'Unknown error occurred' + ); + process.exit(1); + } +} + +/** + * Shorthand: auto-launch a session if needed, then execute an agent-browser command. + * Enables `firecrawl browser "open example.com"` without a separate launch step. + */ +export async function handleBrowserQuickExecute( + options: BrowserQuickExecuteOptions +): Promise { + // Auto-launch if no active session + const existing = loadBrowserSession(); + if (!existing) { + try { + const app = getClient({ apiKey: options.apiKey, apiUrl: options.apiUrl }); + const data = await app.browser({}); + + if (!data.success) { + console.error('Error:', data.error || 'Failed to launch session'); + process.exit(1); + } + + saveBrowserSession({ + id: data.id!, + cdpUrl: data.cdpUrl!, + createdAt: new Date().toISOString(), + }); + + // Print session info to stderr so it doesn't mix with command output + console.error(`Session launched: ${data.id}`); + } catch (error) { + console.error( + 'Error:', + error instanceof Error ? error.message : 'Failed to launch session' + ); + process.exit(1); + } + } + + // Auto-prefix agent-browser if needed + let finalCode = options.code; + if (!finalCode.startsWith('agent-browser')) { + finalCode = `agent-browser ${finalCode}`; + } + + // Execute via the standard handler + await handleBrowserExecute({ + code: finalCode, + language: 'bash', + apiKey: options.apiKey, + apiUrl: options.apiUrl, + output: options.output, + json: options.json, + }); +} + +/** + * Close a browser session + */ +export async function handleBrowserClose( + options: BrowserCloseOptions +): Promise { + try { + const sessionId = getSessionId(options.session); + const app = getClient({ apiKey: options.apiKey, apiUrl: options.apiUrl }); + + const data = await app.deleteBrowser(sessionId); + + if (!data.success) { + console.error('Error:', data.error || 'Unknown error'); + process.exit(1); + } + + // Clear stored session + const stored = loadBrowserSession(); + if (stored && stored.id === sessionId) { + clearBrowserSession(); + } + + console.log(`Session closed (${sessionId})`); + + if (options.json) { + const output = JSON.stringify({ success: true, id: sessionId }, null, 2); + writeOutput(output, options.output, !!options.output); + } + } catch (error) { + console.error( + 'Error:', + error instanceof Error ? error.message : 'Unknown error occurred' + ); + process.exit(1); + } +} diff --git a/src/index.ts b/src/index.ts index c36c9dff7d..77fb79d36f 100644 --- a/src/index.ts +++ b/src/index.ts @@ -15,6 +15,13 @@ import { handleCrawlCommand } from './commands/crawl'; import { handleMapCommand } from './commands/map'; import { handleSearchCommand } from './commands/search'; import { handleAgentCommand } from './commands/agent'; +import { + handleBrowserLaunch, + handleBrowserExecute, + handleBrowserList, + handleBrowserClose, + handleBrowserQuickExecute, +} from './commands/browser'; import { handleVersionCommand } from './commands/version'; import { handleLoginCommand } from './commands/login'; import { handleLogoutCommand } from './commands/logout'; @@ -37,6 +44,7 @@ const AUTH_REQUIRED_COMMANDS = [ 'map', 'search', 'agent', + 'browser', 'credit-usage', ]; @@ -580,11 +588,287 @@ function createAgentCommand(): Command { return agentCmd; } -// Add crawl, map, search, and agent commands to main program +/** + * Create and configure the browser command + */ +function createBrowserCommand(): Command { + const browserCmd = new Command('browser') + .description( + 'Launch cloud browser sessions and execute Python, JavaScript, or bash code remotely via Playwright' + ) + .argument('[code]', 'Shorthand: auto-launch session + execute command') + .option( + '-k, --api-key ', + 'Firecrawl API key (overrides global --api-key)' + ) + .option('--api-url ', 'API URL (overrides global --api-url)') + .option('-o, --output ', 'Output file path (default: stdout)') + .option('--json', 'Output as JSON format', false) + .action(async (code, options) => { + if (code) { + await handleBrowserQuickExecute({ + code, + apiKey: options.apiKey, + apiUrl: options.apiUrl, + output: options.output, + json: options.json, + }); + } + }) + .addHelpText( + 'after', + ` +Shorthand (auto-launches session if needed): + $ firecrawl browser "open https://example.com" + $ firecrawl browser "snapshot" + $ firecrawl browser "click @e5" + $ firecrawl browser "scrape" + +Explicit subcommands: + $ firecrawl browser launch-session + $ firecrawl browser execute "open https://example.com" + $ firecrawl browser list active + $ firecrawl browser close + + By default, commands are sent to agent-browser (pre-installed in every sandbox). + Use --python or --node to run Playwright code instead. + $ firecrawl browser execute --python 'print(await page.title())' + $ firecrawl browser execute --node 'await page.title()' + + See all agent-browser commands: + $ firecrawl browser execute "--help" +` + ); + + browserCmd + .command('launch-session') + .description( + 'Launch a new cloud browser session (without executing a command)' + ) + .option( + '--ttl ', + 'Total session TTL in seconds (default: 300)', + parseInt + ) + .option('--ttl-inactivity ', 'Inactivity TTL in seconds', parseInt) + .option('--stream', 'Enable live view streaming', false) + .option( + '-k, --api-key ', + 'Firecrawl API key (overrides global --api-key)' + ) + .option('--api-url ', 'API URL (overrides global --api-url)') + .option('-o, --output ', 'Output file path (default: stdout)') + .option('--json', 'Output as JSON format', false) + .addHelpText( + 'after', + ` +Output: + Prints the Session ID and CDP URL. The session is auto-saved so + subsequent execute/close commands target it automatically. + + Tip: Use the shorthand to launch + execute in one step: + $ firecrawl browser "open https://example.com" + +Examples: + $ firecrawl browser launch-session + $ firecrawl browser launch-session --stream --ttl 600 + $ firecrawl browser launch-session --ttl 300 --ttl-inactivity 60 + $ firecrawl browser launch-session -o session.json --json +` + ) + .action(async (options) => { + await handleBrowserLaunch({ + ttl: options.ttl, + ttlInactivity: options.ttlInactivity, + stream: options.stream, + apiKey: options.apiKey, + apiUrl: options.apiUrl, + output: options.output, + json: options.json, + }); + }); + + browserCmd + .command('execute') + .description( + 'Execute agent-browser commands (default), or Playwright Python/JS in a session' + ) + .argument( + '', + 'agent-browser command (default) or Playwright code (with --python/--node)' + ) + .option('--python', 'Execute as Playwright Python code', false) + .option('--node', 'Execute as Playwright JavaScript code', false) + .option( + '--bash', + 'Execute bash in the sandbox (agent-browser pre-installed, CDP_URL auto-injected)', + false + ) + .option( + '--session ', + 'Session ID (default: active session from last launch)' + ) + .option( + '-k, --api-key ', + 'Firecrawl API key (overrides global --api-key)' + ) + .option('--api-url ', 'API URL (overrides global --api-url)') + .option('-o, --output ', 'Output file path (default: stdout)') + .option('--json', 'Output as JSON format', false) + .addHelpText( + 'after', + ` +How it works: + By default, commands are sent to agent-browser (pre-installed in every sandbox). + You don't need to type "agent-browser" — it's added automatically. + +agent-browser examples (default): + $ firecrawl browser execute "open https://example.com" + $ firecrawl browser execute "snapshot" + $ firecrawl browser execute "click @e5" + $ firecrawl browser execute "scrape" + + You can still pass the full command if you prefer: + $ firecrawl browser execute "agent-browser snapshot" + + Use --bash for arbitrary bash commands (not just agent-browser): + $ firecrawl browser execute --bash 'ls /tmp' + +Python examples (use --python): + $ firecrawl browser execute --python 'print(await page.title())' + $ firecrawl browser execute --python ' + await page.goto("https://news.ycombinator.com") + title = await page.title() + items = await page.query_selector_all(".titleline > a") + for item in items[:5]: + print(await item.inner_text()) + ' + +JavaScript examples (use --node): + $ firecrawl browser execute --node 'await page.goto("https://example.com"); await page.title()' + +Target a specific session: + $ firecrawl browser execute --session "snapshot" + +Note: --python, --node, and --bash are mutually exclusive. +` + ) + .action(async (code, options) => { + const flagCount = [options.python, options.node, options.bash].filter( + Boolean + ).length; + if (flagCount > 1) { + console.error( + 'Error: Only one of --python, --node, or --bash can be specified' + ); + process.exit(1); + } + const language = options.python + ? 'python' + : options.node + ? 'node' + : 'bash'; + + // In default/bash mode, auto-prefix "agent-browser" if not already present + let finalCode = code; + if ( + language === 'bash' && + !options.bash && + !finalCode.startsWith('agent-browser') + ) { + finalCode = `agent-browser ${finalCode}`; + } + + await handleBrowserExecute({ + code: finalCode, + language, + session: options.session, + apiKey: options.apiKey, + apiUrl: options.apiUrl, + output: options.output, + json: options.json, + }); + }); + + browserCmd + .command('list [status]') + .description( + 'List browser sessions (optionally filter by: active, destroyed)' + ) + .option( + '-k, --api-key ', + 'Firecrawl API key (overrides global --api-key)' + ) + .option('--api-url ', 'API URL (overrides global --api-url)') + .option('-o, --output ', 'Output file path (default: stdout)') + .option('--json', 'Output as JSON format', false) + .addHelpText( + 'after', + ` +Examples: + $ firecrawl browser list + $ firecrawl browser list active + $ firecrawl browser list destroyed + $ firecrawl browser list --json +` + ) + .action(async (status, options) => { + if (status && !['active', 'destroyed'].includes(status)) { + console.error( + `Error: Invalid status "${status}". Use "active" or "destroyed".` + ); + process.exit(1); + } + await handleBrowserList({ + status, + apiKey: options.apiKey, + apiUrl: options.apiUrl, + output: options.output, + json: options.json, + }); + }); + + browserCmd + .command('close') + .description('Close a browser session') + .option( + '--session ', + 'Session ID (default: active session from last launch)' + ) + .option( + '-k, --api-key ', + 'Firecrawl API key (overrides global --api-key)' + ) + .option('--api-url ', 'API URL (overrides global --api-url)') + .option('-o, --output ', 'Output file path (default: stdout)') + .option('--json', 'Output as JSON format', false) + .addHelpText( + 'after', + ` +Examples: + $ firecrawl browser close + $ firecrawl browser close --session +` + ) + .action(async (options) => { + await handleBrowserClose({ + session: options.session, + apiKey: options.apiKey, + apiUrl: options.apiUrl, + output: options.output, + json: options.json, + }); + }); + + return browserCmd; +} + +// Add crawl, map, search, agent, and browser commands to main program program.addCommand(createCrawlCommand()); program.addCommand(createMapCommand()); program.addCommand(createSearchCommand()); program.addCommand(createAgentCommand()); +program.addCommand(createBrowserCommand()); program .command('config') diff --git a/src/utils/browser-session.ts b/src/utils/browser-session.ts new file mode 100644 index 0000000000..af4ccefb15 --- /dev/null +++ b/src/utils/browser-session.ts @@ -0,0 +1,107 @@ +/** + * Browser session persistence utility + * Stores active browser session info in platform-specific config directory + */ + +import * as fs from 'fs'; +import * as path from 'path'; +import * as os from 'os'; + +export interface StoredBrowserSession { + id: string; + cdpUrl: string; + createdAt: string; +} + +/** + * Get the platform-specific config directory + */ +function getConfigDir(): string { + const homeDir = os.homedir(); + const platform = os.platform(); + + switch (platform) { + case 'darwin': + return path.join( + homeDir, + 'Library', + 'Application Support', + 'firecrawl-cli' + ); + case 'win32': + return path.join(homeDir, 'AppData', 'Roaming', 'firecrawl-cli'); + default: + return path.join(homeDir, '.config', 'firecrawl-cli'); + } +} + +/** + * Get the browser session file path + */ +function getSessionPath(): string { + return path.join(getConfigDir(), 'browser-session.json'); +} + +/** + * Ensure the config directory exists + */ +function ensureConfigDir(): void { + const configDir = getConfigDir(); + if (!fs.existsSync(configDir)) { + fs.mkdirSync(configDir, { recursive: true, mode: 0o700 }); + } +} + +/** + * Save active browser session to disk + */ +export function saveBrowserSession(session: StoredBrowserSession): void { + ensureConfigDir(); + const sessionPath = getSessionPath(); + fs.writeFileSync(sessionPath, JSON.stringify(session, null, 2), 'utf-8'); +} + +/** + * Load stored browser session from disk + */ +export function loadBrowserSession(): StoredBrowserSession | null { + try { + const sessionPath = getSessionPath(); + if (!fs.existsSync(sessionPath)) { + return null; + } + const data = fs.readFileSync(sessionPath, 'utf-8'); + return JSON.parse(data) as StoredBrowserSession; + } catch { + return null; + } +} + +/** + * Clear stored browser session from disk + */ +export function clearBrowserSession(): void { + try { + const sessionPath = getSessionPath(); + if (fs.existsSync(sessionPath)) { + fs.unlinkSync(sessionPath); + } + } catch { + // Ignore errors + } +} + +/** + * Resolve session ID from override flag or stored session + */ +export function getSessionId(overrideId?: string): string { + if (overrideId) return overrideId; + + const stored = loadBrowserSession(); + if (stored) return stored.id; + + throw new Error( + 'No active browser session. Launch one with: firecrawl browser launch\n' + + 'Or specify a session ID with: --session ' + ); +}