From f64e106d6aabb8a5bd27b6b182c76a96fce85fa1 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 23 Sep 2026 13:18:50 +0000 Subject: [PATCH] docs: add agent quickstart files for all SDK languages Generate one canonical agent quickstart document per SDK language (Node.js, Python, Rust, Java, Elixir) covering search, scrape, and interact endpoints with confirmed parameters from SDK source and OpenAPI spec. Co-Authored-By: Claude Opus 4.6 (1M context) Claude-Session: https://claude.ai/code/session_01JP6oSmKcg4kEKAwzWrgp4Q --- agent-quickstart/elixir.mdx | 207 +++++++++++++++++++++++++++++++ agent-quickstart/java.mdx | 237 +++++++++++++++++++++++++++++++++++ agent-quickstart/node.mdx | 203 ++++++++++++++++++++++++++++++ agent-quickstart/python.mdx | 199 ++++++++++++++++++++++++++++++ agent-quickstart/rust.mdx | 239 ++++++++++++++++++++++++++++++++++++ 5 files changed, 1085 insertions(+) create mode 100644 agent-quickstart/elixir.mdx create mode 100644 agent-quickstart/java.mdx create mode 100644 agent-quickstart/node.mdx create mode 100644 agent-quickstart/python.mdx create mode 100644 agent-quickstart/rust.mdx diff --git a/agent-quickstart/elixir.mdx b/agent-quickstart/elixir.mdx new file mode 100644 index 000000000..d723db239 --- /dev/null +++ b/agent-quickstart/elixir.mdx @@ -0,0 +1,207 @@ +--- +title: "Elixir Agent Quickstart" +description: "Canonical Firecrawl Elixir quickstart for external agents using search, scrape, and interact." +--- + +# Firecrawl Elixir Agent Quickstart + +Canonical quickstart for external agents. Generated from `:firecrawl` hex package v1.11.0 source (`firecrawl/apps/elixir-sdk`) and the v2 OpenAPI spec. Function names are auto-generated from the OpenAPI spec; parameters use snake_case and are converted to camelCase for the API. + +## Install + +```elixir +# mix.exs +defp deps do + [ + {:firecrawl, "~> 1.11"} + ] +end +``` + +## Authenticate + +```elixir +# Option 1: Application config +# config/config.exs +config :firecrawl, api_key: "fc-your-api-key" + +# Option 2: Per-call option +Firecrawl.search_and_scrape([query: "test"], api_key: "fc-your-api-key") +``` + +Additional per-call options: `base_url` (override default `https://api.firecrawl.dev/v2`), plus any keyword passed through to `Req`. + +## When To Use What + +- `search_and_scrape` — start with a query, discover relevant pages, then scrape or interact with the results. +- `scrape_and_extract_from_url` — you already have a URL and want structured page content. +- `interact_with_scrape_browser_session` — the page needs code execution in a post-scrape browser session. + +## Search + +### Why use it + +Discover relevant pages from a text query. Supports `site:` scoping, source filtering, category narrowing, and inline scraping of results. + +### Preferred SDK function + +`Firecrawl.search_and_scrape(params, opts \\ [])` → `{:ok, %Req.Response{}}` or `{:error, exception}` + +Bang variant: `Firecrawl.search_and_scrape!(params, opts)` → `%Req.Response{}` or raises. + +### Example + +```elixir +{:ok, response} = Firecrawl.search_and_scrape( + query: "site:docs.firecrawl.dev webhook retries", + sources: ["web"], + limit: 5, + scrape_options: [formats: ["markdown"], only_main_content: true] +) + +web_results = response.body["data"]["web"] || [] +Enum.each(web_results, fn hit -> + IO.puts("#{hit["url"]} #{hit["title"]}") +end) +``` + +### Parameters + +First argument (keyword list): + +| Parameter | Type | Description | +|---|---|---| +| `query` | `:string` | Search query (required). Use `site:example.com` to limit by domain. | +| `sources` | `{:list, :any}` | Sources: `"web"`, `"news"`, `"images"`. | +| `categories` | `{:list, :any}` | Filter by category. | +| `include_domains` | `{:list, :string}` | Only include these domains. | +| `exclude_domains` | `{:list, :string}` | Exclude these domains. | +| `limit` | `:integer` | Max results. | +| `tbs` | `:string` | Time-based filter (e.g. `"qdr:d"`). | +| `location` | `:string` | Geographic location. | +| `country` | `:string` | ISO country code. | +| `ignore_invalid_urls` | `:boolean` | Drop unscrappable URLs. | +| `timeout` | `:integer` | Timeout in milliseconds. | +| `highlights` | `:boolean` | Query-relevant highlights (default true). | +| `scrape_options` | `:keyword_list` | Scrape each result (see Scrape parameters). | +| `enterprise` | `{:list, :string}` | ZDR options: `["zdr"]` or `["anon"]`. | + +## Scrape + +### Why use it + +Get structured content from a URL in one or more formats: markdown, HTML, JSON extraction, screenshots, and more. + +### Preferred SDK function + +`Firecrawl.scrape_and_extract_from_url(params, opts \\ [])` → `{:ok, %Req.Response{}}` or `{:error, exception}` + +Bang variant: `Firecrawl.scrape_and_extract_from_url!(params, opts)` → `%Req.Response{}` or raises. + +### Example + +```elixir +{:ok, response} = Firecrawl.scrape_and_extract_from_url( + url: "https://example.com/pricing", + formats: ["markdown"], + only_main_content: true +) + +IO.puts(response.body["data"]["markdown"]) +``` + +### Parameters + +First argument (keyword list): + +| Parameter | Type | Description | +|---|---|---| +| `url` | `:string` | URL to scrape (required). | +| `formats` | `{:list, :any}` | Output formats: `"markdown"`, `"html"`, `"rawHtml"`, `"links"`, `"images"`, `"screenshot"`, `"json"`, `"audio"`, `"video"`, etc. | +| `headers` | `:any` | Custom HTTP headers. | +| `include_tags` | `{:list, :string}` | Include only these HTML tags. | +| `exclude_tags` | `{:list, :string}` | Exclude these HTML tags. | +| `only_main_content` | `:boolean` | Strip nav, footer, boilerplate. | +| `timeout` | `:integer` | Timeout in ms (default 60000, max 300000). | +| `wait_for` | `:integer` | Wait for page render (ms). | +| `mobile` | `:boolean` | Use mobile viewport. | +| `parsers` | `{:list, :any}` | File parsing controls (e.g. PDF). | +| `actions` | `{:list, :any}` | Pre-scrape browser actions. | +| `location` | `:keyword_list` | Geo/language: `[country: "US", languages: ["en-US"]]`. | +| `skip_tls_verification` | `:boolean` | Skip TLS verification. | +| `remove_base64_images` | `:boolean` | Drop base64 images. | +| `block_ads` | `:boolean` | Block ads and cookie popups. | +| `proxy` | `:basic \| :enhanced \| :auto` | Proxy type. | +| `max_age` | `:integer` | Cached data max age (ms, default 2 days). | +| `min_age` | `:integer` | Cache-only mode, min age (ms). | +| `store_in_cache` | `:boolean` | Cache the result. | +| `lockdown` | `:boolean` | Only serve cached results. | +| `redact_pii` | `:boolean` | Redact PII. | +| `audit_metadata` | `:keyword_list` | SIEM logging: `[username: "agent-1"]`. | +| `profile` | `:keyword_list` | Persistent browser profile: `[name: "session", save_changes: true]`. | + +## Interact + +### Why use it + +Execute code in the browser session tied to a scrape job for post-scrape automation. + +### Preferred SDK function + +`Firecrawl.interact_with_scrape_browser_session(job_id, params, opts \\ [])` → `{:ok, %Req.Response{}}` or `{:error, exception}` + +Bang variant: `Firecrawl.interact_with_scrape_browser_session!(job_id, params, opts)` → `%Req.Response{}` or raises. + +### Example + +```elixir +{:ok, scrape_response} = Firecrawl.scrape_and_extract_from_url( + url: "https://example.com", + formats: ["markdown"] +) + +job_id = scrape_response.body["data"]["metadata"]["scrapeId"] + +{:ok, result} = Firecrawl.interact_with_scrape_browser_session(job_id, + code: "console.log(await page.title());", + language: :node +) + +IO.puts(result.body["stdout"]) + +# Stop the session: +Firecrawl.stop_interactive_scrape_browser_session(job_id) +``` + +### Parameters + +First argument: `job_id` (String.t) — scrape job ID. + +Second argument (keyword list): + +| Parameter | Type | Description | +|---|---|---| +| `code` | `:string` | Code to execute in the browser session (required). | +| `language` | `:python \| :node \| :bash` | Runtime for code execution. | +| `timeout` | `:integer` | Execution timeout in seconds. | +| `origin` | `:string` | Origin label for telemetry. | + +### Stop session + +`Firecrawl.stop_interactive_scrape_browser_session(job_id, opts \\ [])` → `{:ok, %Req.Response{}}` or `{:error, exception}` + +## Notes + +- Function names are auto-generated from the OpenAPI spec, so they are verbose: `search_and_scrape`, `scrape_and_extract_from_url`, `interact_with_scrape_browser_session`. +- Every function has a bang (`!`) variant that raises on error. +- Parameters use `snake_case` keys; they are converted to `camelCase` for the JSON body automatically. +- The last argument to every function is always `opts \\ []` for `:api_key`, `:base_url`, and Req pass-through options. +- Responses are raw `%Req.Response{}` structs; access data via `response.body["data"]`. +- The Elixir SDK's `interact_with_scrape_browser_session` requires `code`; it does not support `prompt`-based natural-language interaction. +- Parameters are validated at call time by `NimbleOptions`. + +## Source Of Truth + +- `firecrawl/apps/elixir-sdk/lib/firecrawl.ex` +- `firecrawl/apps/elixir-sdk/mix.exs` +- `firecrawl-docs/api-reference/v2-openapi.json` diff --git a/agent-quickstart/java.mdx b/agent-quickstart/java.mdx new file mode 100644 index 000000000..65512d9c6 --- /dev/null +++ b/agent-quickstart/java.mdx @@ -0,0 +1,237 @@ +--- +title: "Java Agent Quickstart" +description: "Canonical Firecrawl Java quickstart for external agents using search, scrape, and interact." +--- + +# Firecrawl Java Agent Quickstart + +Canonical quickstart for external agents. Generated from `firecrawl-java` v1.18.0 source (`firecrawl/apps/java-sdk`) and the v2 OpenAPI spec. Method names and parameters match the SDK public API. + +## Install + +Maven: + +```xml + + com.firecrawl + firecrawl-java + 1.18.0 + +``` + +Gradle: + +```groovy +implementation 'com.firecrawl:firecrawl-java:1.18.0' +``` + +Requires Java 11+. + +## Authenticate + +```java +import com.firecrawl.client.FirecrawlClient; + +FirecrawlClient client = FirecrawlClient.builder() + .apiKey("fc-your-api-key") + .build(); + +// Or read from FIRECRAWL_API_KEY env var / firecrawl.apiKey system property: +FirecrawlClient client = FirecrawlClient.fromEnv(); +``` + +Builder options: `apiKey` (String), `apiUrl` (String, default `"https://api.firecrawl.dev"`), `timeoutMs` (long, default 300000), `maxRetries` (int, default 3), `backoffFactor` (double, default 0.5), `httpClient` (OkHttpClient), `asyncExecutor` (Executor). + +## When To Use What + +- `search` — start with a query, discover relevant pages, then scrape or interact with the results. +- `scrape` — you already have a URL and want structured page content. +- `interact` — the page needs clicks, form fills, or post-scrape browser actions via code execution. + +## Search + +### Why use it + +Discover relevant pages from a text query. Supports `site:` scoping, source filtering, category narrowing, and inline scraping of results. + +### Preferred SDK method + +`client.search(query, options)` → `SearchData` + +### Example + +```java +import com.firecrawl.models.SearchOptions; +import com.firecrawl.models.SearchData; +import com.firecrawl.models.ScrapeOptions; +import java.util.List; + +SearchData results = client.search("site:docs.firecrawl.dev webhook retries", + SearchOptions.builder() + .sources(List.of("web")) + .limit(5) + .scrapeOptions(ScrapeOptions.builder() + .formats(List.of("markdown")) + .onlyMainContent(true) + .build()) + .build()); + +for (var hit : results.getWeb()) { + System.out.println(hit.get("url") + " " + hit.get("title")); +} +``` + +### Parameters + +`SearchOptions` fields (built via `SearchOptions.builder()`): + +| Field | Type | Description | +|---|---|---| +| `sources` | `List` | Sources: `"web"`, `"news"`, `"images"`. | +| `categories` | `List` | Filter: `"github"`, `"research"`, `"pdf"`. | +| `includeDomains` | `List` | Only include these domains. | +| `excludeDomains` | `List` | Exclude these domains. | +| `limit` | `Integer` | Max results. | +| `tbs` | `String` | Time-based filter (e.g. `"qdr:d"`). | +| `location` | `String` | Location for geo-targeted results. | +| `country` | `String` | Country code (e.g. `"us"`). | +| `ignoreInvalidURLs` | `Boolean` | Drop unscrappable URLs. | +| `timeout` | `Integer` | Timeout in milliseconds. | +| `highlights` | `Boolean` | Query-relevant highlights (default true on server). | +| `scrapeOptions` | `ScrapeOptions` | Scrape each result (see Scrape parameters). | +| `integration` | `String` | Integration identifier. | + +### Return value + +`SearchData` with `getWeb()`, `getNews()`, `getImages()` returning `List>`. + +## Scrape + +### Why use it + +Get structured content from a URL in one or more formats: markdown, HTML, JSON extraction, screenshots, and more. + +### Preferred SDK method + +`client.scrape(url, options)` → `Document` + +Overloads: `client.scrape(url)` (no options), `client.scrapeAsync(url, options)` → `CompletableFuture`. + +### Example + +```java +import com.firecrawl.models.ScrapeOptions; +import com.firecrawl.models.Document; +import java.util.List; +import java.util.Map; + +Document doc = client.scrape("https://example.com/pricing", + ScrapeOptions.builder() + .formats(List.of( + "markdown", + Map.of("type", "json", "prompt", "Extract plan names and prices.") + )) + .onlyMainContent(true) + .build()); + +System.out.println(doc.getMarkdown()); +System.out.println(doc.getJson()); +``` + +### Parameters + +`ScrapeOptions` fields (built via `ScrapeOptions.builder()`): + +| Field | Type | Description | +|---|---|---| +| `formats` | `List` | Output formats. Strings: `"markdown"`, `"html"`, `"rawHtml"`, `"links"`, `"screenshot"`, `"json"`, `"audio"`, `"video"`. Object forms via `Map`: `{"type": "json", "prompt": ..., "schema": ...}`, etc. | +| `headers` | `Map` | Custom HTTP headers. | +| `includeTags` | `List` | Include only these HTML tags. | +| `excludeTags` | `List` | Exclude these HTML tags. | +| `onlyMainContent` | `Boolean` | Strip nav, footer, boilerplate. | +| `timeout` | `Integer` | Timeout in milliseconds. | +| `waitFor` | `Integer` | Wait for page render (ms). | +| `mobile` | `Boolean` | Use mobile viewport. | +| `parsers` | `List` | File parsing (e.g. `"pdf"` or `PdfParser`). | +| `actions` | `List>` | Pre-scrape browser actions. | +| `location` | `LocationConfig` | Geo/language: `.country(String)`, `.languages(List)`. | +| `skipTlsVerification` | `Boolean` | Skip TLS verification. | +| `removeBase64Images` | `Boolean` | Drop base64 images. | +| `blockAds` | `Boolean` | Block ads and cookie popups. | +| `proxy` | `String` | `"basic"`, `"stealth"`, `"enhanced"`, `"auto"`. | +| `maxAge` | `Long` | Cached data max age (ms). | +| `storeInCache` | `Boolean` | Cache the result. | +| `lockdown` | `Boolean` | Only serve cached results. | +| `redactPII` | `Boolean` | Redact PII. | +| `auditMetadata` | `AuditMetadata` | SIEM logging: takes `username`. | +| `integration` | `String` | Integration identifier. | + +## Interact + +### Why use it + +Execute code in the browser session tied to a scrape job for post-scrape automation. + +### Preferred SDK method + +`client.interact(jobId, code)` → `BrowserExecuteResponse` + +Overloads: +- `interact(jobId, code)` — defaults: language `"node"`, no timeout. +- `interact(jobId, code, language, timeout)` — specify language and timeout. +- `interact(jobId, code, language, timeout, origin)` — full control. + +Async: `client.interactAsync(...)` → `CompletableFuture`. + +### Example + +```java +import com.firecrawl.models.Document; +import com.firecrawl.models.ScrapeOptions; +import com.firecrawl.models.BrowserExecuteResponse; +import java.util.List; + +Document doc = client.scrape("https://example.com", + ScrapeOptions.builder().formats(List.of("markdown")).build()); + +String jobId = (String) doc.getMetadata().get("scrapeId"); + +BrowserExecuteResponse result = client.interact(jobId, + "console.log(await page.title());", + "node", + 60); +System.out.println(result.getStdout()); + +// Stop the session: +client.stopInteractiveBrowser(jobId); +``` + +### Parameters + +| Parameter | Type | Description | +|---|---|---| +| `jobId` | `String` | Scrape job ID from `document.metadata.scrapeId`. | +| `code` | `String` | Code to execute in the browser session. | +| `language` | `String` | Runtime: `"python"`, `"node"`, `"bash"` (default `"node"`). | +| `timeout` | `Integer` | Execution timeout in seconds (1-300, default 30 on API). | +| `origin` | `String` | Origin label for telemetry. | + +### Stop session + +`client.stopInteractiveBrowser(jobId)` → `BrowserDeleteResponse` + +## Notes + +- The Java SDK uses `camelCase` parameter names matching the API directly. +- Deprecated aliases: `scrapeExecute` → `interact`, `deleteScrapeBrowser` → `stopInteractiveBrowser`. +- The Java SDK's `interact` method requires `code`; it does not support `prompt`-based natural-language interaction. +- All sync methods have `*Async` counterparts returning `CompletableFuture`. +- `SearchData` returns `List>` rather than typed model objects for search results. + +## Source Of Truth + +- `firecrawl/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlClient.java` +- `firecrawl/apps/java-sdk/src/main/java/com/firecrawl/models/ScrapeOptions.java` +- `firecrawl/apps/java-sdk/src/main/java/com/firecrawl/models/SearchOptions.java` +- `firecrawl/apps/java-sdk/build.gradle.kts` +- `firecrawl-docs/api-reference/v2-openapi.json` diff --git a/agent-quickstart/node.mdx b/agent-quickstart/node.mdx new file mode 100644 index 000000000..33119a409 --- /dev/null +++ b/agent-quickstart/node.mdx @@ -0,0 +1,203 @@ +--- +title: "Node.js Agent Quickstart" +description: "Canonical Firecrawl Node.js quickstart for external agents using search, scrape, and interact." +--- + +# Firecrawl Node.js Agent Quickstart + +Canonical quickstart for external agents. Generated from `firecrawl` SDK v4.41.0 source (`firecrawl/apps/js-sdk/firecrawl`) and the v2 OpenAPI spec. Method names and parameters match the SDK public API. + +## Install + +```bash +npm install firecrawl +``` + +## Authenticate + +```ts +import { Firecrawl } from "firecrawl"; + +const client = new Firecrawl({ + apiKey: process.env.FIRECRAWL_API_KEY, + // apiUrl: "https://api.firecrawl.dev" // optional; falls back to FIRECRAWL_API_URL env var or cloud default +}); +``` + +Constructor options: `apiKey` (string), `apiUrl` (string), `timeoutMs` (number), `maxRetries` (number), `backoffFactor` (number). Passing a plain string treats it as the API key: `new Firecrawl("fc-...")`. + +## When To Use What + +- `search` — start with a query, discover relevant pages, then scrape or interact with the results. +- `scrape` — you already have a URL and want structured page content. +- `interact` — the page needs clicks, form fills, or post-scrape browser actions via code or natural language. + +## Search + +### Why use it + +Discover relevant pages from a text query. Supports `site:` scoping, source filtering, category narrowing, and inline scraping of results. + +### Preferred SDK method + +`client.search(query, options?)` → `Promise` + +### Example + +```ts +const results = await client.search("site:docs.firecrawl.dev webhook retries", { + sources: ["web"], + limit: 5, + scrapeOptions: { + formats: ["markdown"], + onlyMainContent: true, + }, +}); + +for (const hit of results.web ?? []) { + console.log(hit.url, hit.title); +} +``` + +### Parameters + +| Parameter | Type | Description | +|---|---|---| +| `query` | `string` | Search query. Use `site:example.com` to limit by domain. | +| `options.sources` | `("web" \| "news" \| "images" \| "alexandria")[]` | Which sources to search. | +| `options.categories` | `("github" \| "research" \| "pdf" \| "developer")[]` | Narrow results by category. | +| `options.includeDomains` | `string[]` | Only include these domains. | +| `options.excludeDomains` | `string[]` | Exclude these domains. | +| `options.limit` | `number` | Max results to return. | +| `options.tbs` | `string` | Time-based filter (e.g. `qdr:d` for past day, `qdr:w` for past week). | +| `options.location` | `string` | Location for geo-targeted results. | +| `options.country` | `string` | ISO 3166-1 alpha-2 country code. | +| `options.ignoreInvalidURLs` | `boolean` | Drop URLs that cannot be scraped. | +| `options.timeout` | `number` | Timeout in milliseconds. | +| `options.highlights` | `boolean` | Return query-relevant highlights (default `true`). | +| `options.scrapeOptions` | `ScrapeOptions` | Scrape each result (see Scrape parameters). | +| `options.domainTools` | `boolean` | Include domain-matched Alexandria tools. | +| `options.toolDetail` | `"compact" \| "summary" \| "full"` | Tool detail level. | + +### Return value + +`SearchData` with optional arrays: `web`, `news`, `images`, `tools`. Access results via `result.web`, not `result.data`. + +## Scrape + +### Why use it + +Get structured content from a URL in one or more formats: markdown, HTML, JSON extraction, screenshots, and more. + +### Preferred SDK method + +`client.scrape(url, options?)` → `Promise` + +### Example + +```ts +const doc = await client.scrape("https://example.com/pricing", { + formats: [ + "markdown", + { type: "json", prompt: "Extract plan names and prices." }, + ], + onlyMainContent: true, +}); + +console.log(doc.markdown); +console.log(doc.json); +``` + +### Parameters + +| Parameter | Type | Description | +|---|---|---| +| `url` | `string` | URL to scrape. | +| `options.formats` | `FormatOption[]` | Output formats. Plain strings: `"markdown"`, `"html"`, `"rawHtml"`, `"links"`, `"images"`, `"screenshot"`, `"summary"`, `"changeTracking"`, `"attributes"`, `"branding"`, `"audio"`, `"video"`. Object forms: `{ type: "json", prompt?, schema? }`, `{ type: "screenshot", fullPage?, quality?, viewport? }`, `{ type: "changeTracking", modes: ("git-diff" \| "json")[], tag? }`, `{ type: "attributes", selectors: [{ selector, attribute }] }`, `{ type: "question", question }`, `{ type: "highlights", query }`. | +| `options.headers` | `Record` | Custom HTTP headers. | +| `options.includeTags` | `string[]` | Include only these HTML tags. | +| `options.excludeTags` | `string[]` | Exclude these HTML tags. | +| `options.onlyMainContent` | `boolean` | Strip nav, footer, and boilerplate. | +| `options.timeout` | `number` | Timeout in milliseconds. | +| `options.waitFor` | `number` | Wait for page render (ms). | +| `options.mobile` | `boolean` | Use mobile viewport. | +| `options.parsers` | `("pdf" \| { type: "pdf", mode?, maxPages? })[]` | File parsing controls. | +| `options.actions` | `Action[]` | Pre-scrape browser actions: `wait`, `click`, `write`, `press`, `scroll`, `screenshot`, `scrape`, `executeJavascript`, `pdf`. | +| `options.location` | `{ country?: string, languages?: string[] }` | Geo/language-aware scraping. | +| `options.skipTlsVerification` | `boolean` | Skip TLS verification. | +| `options.removeBase64Images` | `boolean` | Drop base64 images from output. | +| `options.fastMode` | `boolean` | Faster scrapes with reduced fidelity. | +| `options.blockAds` | `boolean` | Block ads and cookie popups. | +| `options.proxy` | `"basic" \| "stealth" \| "enhanced" \| "auto"` | Proxy control. | +| `options.maxAge` | `number` | Use cached data up to this age (ms). | +| `options.minAge` | `number` | Use cached data only if at least this old (ms). | +| `options.storeInCache` | `boolean` | Cache the result. | +| `options.profile` | `{ name: string, saveChanges?: boolean }` | Persistent browser profile. | +| `options.lockdown` | `boolean` | Only serve cached results. | +| `options.redactPII` | `boolean` | Redact personally identifiable information. | +| `options.auditMetadata` | `{ username: string }` | SIEM logging attribution. | +| `options.domainTools` | `boolean` | Include domain-matched Alexandria tools. | +| `options.toolDetail` | `"compact" \| "summary" \| "full"` | Tool detail level. | + +## Interact + +### Why use it + +Control a browser session tied to a scrape job. Run code or give natural-language instructions to click, fill forms, navigate, or extract data after the initial scrape. + +### Preferred SDK method + +`client.interact(jobId, args)` → `Promise` + +### Example + +```ts +const doc = await client.scrape("https://example.com", { formats: ["markdown"] }); +const jobId = doc.metadata?.scrapeId; +if (!jobId) throw new Error("Missing scrapeId from scrape response"); + +const result = await client.interact(jobId, { + prompt: "Click the pricing tab and summarize the plans.", +}); +console.log(result.output); + +// Or run code directly: +const codeResult = await client.interact(jobId, { + code: "console.log(await page.title());", + language: "node", +}); +console.log(codeResult.stdout); + +// Stop the session when done: +await client.stopInteraction(jobId); +``` + +### Parameters + +| Parameter | Type | Description | +|---|---|---| +| `jobId` | `string` | Scrape job ID from `document.metadata.scrapeId`. | +| `args.code` | `string` | Code to run in the browser session. | +| `args.prompt` | `string` | Natural-language instruction for the browser agent. | +| `args.language` | `"python" \| "node" \| "bash"` | Runtime for code execution (default `"node"`). | +| `args.timeout` | `number` | Execution timeout in seconds. | + +At least one of `code` or `prompt` must be provided. + +### Stop session + +`client.stopInteraction(jobId)` → `Promise` + +## Notes + +- Deprecated aliases: `scrapeExecute` → `interact`, `stopInteractiveBrowser`/`deleteScrapeBrowser` → `stopInteraction`, `scrapeUrl` → `scrape`. +- The default `Firecrawl` export is the v2 client; v1 is available via `client.v1`. +- Zod schemas passed in `formats` (for `json` or `changeTracking`) are converted to JSON Schema by the SDK. +- The package declares Node.js >= 22 in `engines`. + +## Source Of Truth + +- `firecrawl/apps/js-sdk/firecrawl/src/index.ts` +- `firecrawl/apps/js-sdk/firecrawl/src/v2/client.ts` +- `firecrawl/apps/js-sdk/firecrawl/src/v2/types.ts` +- `firecrawl-docs/api-reference/v2-openapi.json` diff --git a/agent-quickstart/python.mdx b/agent-quickstart/python.mdx new file mode 100644 index 000000000..f8a7d0528 --- /dev/null +++ b/agent-quickstart/python.mdx @@ -0,0 +1,199 @@ +--- +title: "Python Agent Quickstart" +description: "Canonical Firecrawl Python quickstart for external agents using search, scrape, and interact." +--- + +# Firecrawl Python Agent Quickstart + +Canonical quickstart for external agents. Generated from `firecrawl-py` SDK source (`firecrawl/apps/python-sdk`) and the v2 OpenAPI spec. Method names and parameters match the SDK public API. + +## Install + +```bash +pip install firecrawl-py +``` + +## Authenticate + +```python +import os +from firecrawl import Firecrawl + +client = Firecrawl(api_key=os.environ.get("FIRECRAWL_API_KEY")) +# Optional: client = Firecrawl(api_key="fc-...", api_url="https://api.firecrawl.dev") +``` + +Constructor parameters: `api_key` (str), `api_url` (str, default `"https://api.firecrawl.dev"`), `timeout` (float), `max_retries` (int, default 3), `backoff_factor` (float, default 0.5). + +An async client is also available: `from firecrawl import AsyncFirecrawl`. + +## When To Use What + +- `search` — start with a query, discover relevant pages, then scrape or interact with the results. +- `scrape` — you already have a URL and want structured page content. +- `interact` — the page needs clicks, form fills, or post-scrape browser actions via code or natural language. + +## Search + +### Why use it + +Discover relevant pages from a text query. Supports `site:` scoping, source filtering, category narrowing, and inline scraping of results. + +### Preferred SDK method + +`client.search(query, **options)` → `SearchData` + +### Example + +```python +results = client.search( + "site:docs.firecrawl.dev webhook retries", + sources=["web"], + limit=5, + scrape_options={"formats": ["markdown"], "only_main_content": True}, +) + +for hit in results.web or []: + print(hit.url, hit.title) +``` + +### Parameters + +| Parameter | Type | Description | +|---|---|---| +| `query` | `str` | Search query. Use `site:example.com` to limit by domain. | +| `sources` | `list[str]` | Sources to search: `"web"`, `"news"`, `"images"`, `"alexandria"`. | +| `categories` | `list[str]` | Narrow by category: `"github"`, `"research"`, `"pdf"`, `"developer"`. | +| `include_domains` | `list[str]` | Only include these domains. | +| `exclude_domains` | `list[str]` | Exclude these domains. | +| `limit` | `int` | Max results to return (default 5). | +| `tbs` | `str` | Time-based filter (e.g. `"qdr:d"` for past day). | +| `location` | `str` | Location for geo-targeted results. | +| `country` | `str` | ISO 3166-1 alpha-2 country code. | +| `ignore_invalid_urls` | `bool` | Drop URLs that cannot be scraped. | +| `timeout` | `int` | Timeout in milliseconds (default 300000). | +| `highlights` | `bool` | Return query-relevant highlights (default `True`). | +| `scrape_options` | `dict \| ScrapeOptions` | Scrape each result (see Scrape parameters). | +| `domain_tools` | `bool` | Include domain-matched Alexandria tools. | +| `tool_detail` | `"compact" \| "summary" \| "full"` | Tool detail level. | + +### Return value + +`SearchData` with optional lists: `web`, `news`, `images`, `tools`. Access results via `result.web`, not `result.data`. + +## Scrape + +### Why use it + +Get structured content from a URL in one or more formats: markdown, HTML, JSON extraction, screenshots, and more. + +### Preferred SDK method + +`client.scrape(url, **options)` → `Document` + +### Example + +```python +doc = client.scrape( + "https://example.com/pricing", + formats=[ + "markdown", + {"type": "json", "prompt": "Extract plan names and prices."}, + ], + only_main_content=True, +) + +print(doc.markdown) +print(doc.json) +``` + +### Parameters + +| Parameter | Type | Description | +|---|---|---| +| `url` | `str` | URL to scrape. | +| `formats` | `list` | Output formats. Strings: `"markdown"`, `"html"`, `"rawHtml"`, `"links"`, `"images"`, `"screenshot"`, `"summary"`, `"changeTracking"`, `"attributes"`, `"branding"`, `"audio"`, `"video"`. Object forms: `{"type": "json", "prompt": ..., "schema": ...}`, `{"type": "screenshot", "fullPage": True}`, `{"type": "changeTracking", "modes": ["git-diff"]}`, `{"type": "attributes", "selectors": [{"selector": ..., "attribute": ...}]}`, `{"type": "question", "question": ...}`, `{"type": "highlights", "query": ...}`. | +| `headers` | `dict[str, str]` | Custom HTTP headers. | +| `include_tags` | `list[str]` | Include only these HTML tags. | +| `exclude_tags` | `list[str]` | Exclude these HTML tags. | +| `only_main_content` | `bool` | Strip nav, footer, and boilerplate. | +| `timeout` | `int` | Timeout in milliseconds. | +| `wait_for` | `int` | Wait for page render (ms). | +| `mobile` | `bool` | Use mobile viewport. | +| `parsers` | `list` | File parsing controls (e.g. `"pdf"` or `{"type": "pdf", "mode": "auto"}`). | +| `actions` | `list[dict]` | Pre-scrape browser actions: `wait`, `click`, `write`, `press`, `scroll`, `screenshot`, `scrape`, `executeJavascript`, `pdf`. | +| `location` | `dict` | Geo/language config: `{"country": "US", "languages": ["en-US"]}`. | +| `skip_tls_verification` | `bool` | Skip TLS verification. | +| `remove_base64_images` | `bool` | Drop base64 images from output. | +| `fast_mode` | `bool` | Faster scrapes with reduced fidelity. | +| `block_ads` | `bool` | Block ads and cookie popups. | +| `proxy` | `str` | Proxy control: `"basic"`, `"stealth"`, `"enhanced"`, `"auto"`. | +| `max_age` | `int` | Use cached data up to this age (ms). | +| `store_in_cache` | `bool` | Cache the result. | +| `profile` | `dict` | Persistent browser profile: `{"name": "my-session", "saveChanges": True}`. | +| `lockdown` | `bool` | Only serve cached results. | +| `redact_pii` | `bool` | Redact personally identifiable information. | +| `audit_metadata` | `dict` | SIEM logging: `{"username": "agent-1"}`. | +| `domain_tools` | `bool` | Include domain-matched Alexandria tools. | +| `tool_detail` | `"compact" \| "summary" \| "full"` | Tool detail level. | + +## Interact + +### Why use it + +Control a browser session tied to a scrape job. Run code or give natural-language instructions to click, fill forms, navigate, or extract data after the initial scrape. + +### Preferred SDK method + +`client.interact(job_id, code=None, *, prompt=None, language="node", timeout=None)` → `BrowserExecuteResponse` + +### Example + +```python +doc = client.scrape("https://example.com", formats=["markdown"]) +job_id = doc.metadata.scrape_id +if not job_id: + raise ValueError("Missing scrapeId from scrape response") + +# Natural-language interaction: +result = client.interact(job_id, prompt="Click the pricing tab and summarize the plans.") +print(result.output) + +# Or run code directly: +code_result = client.interact(job_id, code="console.log(await page.title());", language="node") +print(code_result.stdout) + +# Stop the session when done: +client.stop_interaction(job_id) +``` + +### Parameters + +| Parameter | Type | Description | +|---|---|---| +| `job_id` | `str` | Scrape job ID from `document.metadata.scrape_id`. | +| `code` | `str` | Code to run in the browser session. | +| `prompt` | `str` | Natural-language instruction for the browser agent. | +| `language` | `"python" \| "node" \| "bash"` | Runtime for code execution (default `"node"`). | +| `timeout` | `int` | Execution timeout in seconds (1-300). | + +At least one of `code` or `prompt` must be provided. + +### Stop session + +`client.stop_interaction(job_id)` → `BrowserDeleteResponse` + +## Notes + +- Deprecated aliases: `scrape_execute` → `interact`, `delete_scrape_browser`/`stop_interactive_browser` → `stop_interaction`, `scrape_url` → `scrape`. +- Class aliases: `FirecrawlApp` = `Firecrawl`, `AsyncFirecrawlApp` = `AsyncFirecrawl`. +- Parameters use `snake_case` in Python (e.g. `only_main_content`, `skip_tls_verification`, `scrape_options`). +- The SDK returns Pydantic models (`Document`, `SearchData`, `BrowserExecuteResponse`), not raw dicts. +- The default `Firecrawl` class is the v2 client; v1 is available via `client.v1`. + +## Source Of Truth + +- `firecrawl/apps/python-sdk/firecrawl/client.py` +- `firecrawl/apps/python-sdk/firecrawl/v2/client.py` +- `firecrawl/apps/python-sdk/firecrawl/v2/types.py` +- `firecrawl-docs/api-reference/v2-openapi.json` diff --git a/agent-quickstart/rust.mdx b/agent-quickstart/rust.mdx new file mode 100644 index 000000000..09b6b6e12 --- /dev/null +++ b/agent-quickstart/rust.mdx @@ -0,0 +1,239 @@ +--- +title: "Rust Agent Quickstart" +description: "Canonical Firecrawl Rust quickstart for external agents using search, scrape, and interact." +--- + +# Firecrawl Rust Agent Quickstart + +Canonical quickstart for external agents. Generated from `firecrawl` crate v2.21.0 source (`firecrawl/apps/rust-sdk`) and the v2 OpenAPI spec. Method names and types match the SDK public API. + +## Install + +```toml +# Cargo.toml +[dependencies] +firecrawl = "2" +tokio = { version = "1", features = ["full"] } +``` + +## Authenticate + +```rust +use firecrawl::Client; + +// Cloud (api.firecrawl.dev): +let client = Client::new("fc-your-api-key")?; + +// Self-hosted: +let client = Client::new_selfhosted("https://your-instance.com", Some("fc-your-api-key"))?; +``` + +`Client::new(api_key)` connects to Firecrawl cloud. `Client::new_selfhosted(api_url, api_key)` connects to a self-hosted instance with an optional key. + +## When To Use What + +- `search` — start with a query, discover relevant pages, then scrape or interact with the results. +- `scrape` — you already have a URL and want structured page content. +- `interact` — the page needs clicks, form fills, or post-scrape browser actions via code or natural language. + +## Search + +### Why use it + +Discover relevant pages from a text query. Supports `site:` scoping, source filtering, category narrowing, and inline scraping of results. + +### Preferred SDK method + +`client.search(query, options)` → `Result` + +### Example + +```rust +use firecrawl::{Client, SearchOptions, ScrapeOptions, Format, SearchSource}; + +let client = Client::new(std::env::var("FIRECRAWL_API_KEY")?)?; + +let response = client.search("site:docs.firecrawl.dev webhook retries", Some(SearchOptions { + sources: Some(vec![SearchSource::Web]), + limit: Some(5), + scrape_options: Some(ScrapeOptions { + formats: Some(vec![Format::Markdown]), + only_main_content: Some(true), + ..Default::default() + }), + ..Default::default() +})).await?; + +for hit in response.data.web.unwrap_or_default() { + println!("{:?}", hit); +} +``` + +### Parameters + +`SearchOptions` fields (all `Option`, default `None`): + +| Field | Type | Description | +|---|---|---| +| `limit` | `Option` | Max results (default 5, max 20). | +| `sources` | `Option>` | Sources: `Web`, `News`, `Images`, `Alexandria`. | +| `categories` | `Option>` | Filter: `Github`, `Research`, `Pdf`. | +| `include_domains` | `Option>` | Only include these domains. | +| `exclude_domains` | `Option>` | Exclude these domains. | +| `tbs` | `Option` | Time-based filter (e.g. `"qdr:d"`). | +| `location` | `Option` | Geographic location string. | +| `country` | `Option` | Country code (e.g. `"US"`). | +| `ignore_invalid_urls` | `Option` | Drop unscrappable URLs. | +| `timeout` | `Option` | Timeout in milliseconds. | +| `highlights` | `Option` | Query-relevant highlights (default true). | +| `scrape_options` | `Option` | Scrape each result (see Scrape parameters). | +| `domain_tools` | `Option` | Include domain-matched Alexandria tools. | +| `tool_detail` | `Option` | `Compact`, `Summary`, `Full`. | + +### Return value + +`SearchResponse` with `success: bool`, `data: SearchData` (`web`, `news`, `images`, `tools`), `warning: Option`. + +## Scrape + +### Why use it + +Get structured content from a URL in one or more formats: markdown, HTML, JSON extraction, screenshots, and more. + +### Preferred SDK method + +`client.scrape(url, options)` → `Result` + +### Example + +```rust +use firecrawl::{Client, ScrapeOptions, Format}; + +let client = Client::new(std::env::var("FIRECRAWL_API_KEY")?)?; + +let doc = client.scrape("https://example.com/pricing", Some(ScrapeOptions { + formats: Some(vec![Format::Markdown]), + only_main_content: Some(true), + ..Default::default() +})).await?; + +println!("{}", doc.markdown.unwrap_or_default()); +``` + +### Parameters + +`ScrapeOptions` fields (all `Option`, default `None`): + +| Field | Type | Description | +|---|---|---| +| `formats` | `Option>` | Output formats: `Markdown`, `Html`, `RawHtml`, `Links`, `Images`, `Screenshot`, `Summary`, `ChangeTracking`, `Json`, `Attributes`, `Branding`, `Product`, `Menu`, `Audio`, `Video`, `Question(...)`, `Highlights(...)`, `Query(...)`. | +| `headers` | `Option>` | Custom HTTP headers. | +| `include_tags` | `Option>` | Include only these HTML tags. | +| `exclude_tags` | `Option>` | Exclude these HTML tags. | +| `only_main_content` | `Option` | Strip nav, footer, boilerplate. | +| `timeout` | `Option` | Timeout in milliseconds. | +| `wait_for` | `Option` | Wait for page render (ms). | +| `mobile` | `Option` | Use mobile viewport. | +| `parsers` | `Option>` | File parsing: `Simple("pdf")` or `Pdf { mode, max_pages, ... }`. | +| `actions` | `Option>` | Pre-scrape browser actions: `Wait`, `Click`, `Write`, `Press`, `Scroll`, `Screenshot`, `Scrape`, `ExecuteJavascript`, `Pdf`. | +| `location` | `Option` | `{ country, languages }`. | +| `skip_tls_verification` | `Option` | Skip TLS verification. | +| `remove_base64_images` | `Option` | Drop base64 images. | +| `fast_mode` | `Option` | Faster scrapes, reduced fidelity. | +| `block_ads` | `Option` | Block ads and cookie popups. | +| `proxy` | `Option` | `Basic`, `Stealth`, `Enhanced`, `Auto`. | +| `max_age` | `Option` | Cached data max age (seconds). | +| `min_age` | `Option` | Cached data min age (seconds). | +| `store_in_cache` | `Option` | Cache the result. | +| `lockdown` | `Option` | Only serve cached results. | +| `redact_pii` | `Option` | Redact PII. | +| `audit_metadata` | `Option` | SIEM logging. | +| `profile` | `Option` | Persistent browser profile: `{ name, save_changes }`. | +| `json_options` | `Option` | JSON extraction: `{ schema, system_prompt, prompt, check_prompt_injection }`. | +| `screenshot_options` | `Option` | `{ full_page, quality, viewport }`. | +| `change_tracking_options` | `Option` | `{ modes, schema, prompt, tag }`. | +| `attribute_selectors` | `Option>` | `{ selector, attribute }`. | +| `domain_tools` | `Option` | Include domain-matched Alexandria tools. | +| `tool_detail` | `Option` | `Compact`, `Summary`, `Full`. | + +### Convenience method + +`client.scrape_with_schema(url, schema, prompt)` → `Result` for JSON extraction in one call. + +## Interact + +### Why use it + +Control a browser session tied to a scrape job. Run code or give natural-language instructions to click, fill forms, navigate, or extract data after the initial scrape. + +### Preferred SDK method + +`client.interact(job_id, options)` → `Result` + +### Example + +```rust +use firecrawl::{Client, ScrapeOptions, ScrapeExecuteOptions, Format}; + +let client = Client::new(std::env::var("FIRECRAWL_API_KEY")?)?; + +let doc = client.scrape("https://example.com", Some(ScrapeOptions { + formats: Some(vec![Format::Markdown]), + ..Default::default() +})).await?; + +let job_id = doc.metadata.as_ref() + .and_then(|m| m.scrape_id.as_deref()) + .expect("Missing scrapeId"); + +// Natural-language interaction: +let result = client.interact(job_id, ScrapeExecuteOptions { + prompt: Some("Click the pricing tab and summarize the plans.".into()), + ..Default::default() +}).await?; +println!("{}", result.output.unwrap_or_default()); + +// Or run code directly: +let code_result = client.interact(job_id, ScrapeExecuteOptions { + code: Some("console.log(await page.title());".into()), + language: Some(ScrapeExecuteLanguage::Node), + ..Default::default() +}).await?; +println!("{}", code_result.stdout.unwrap_or_default()); + +// Stop the session: +client.stop_interaction(job_id).await?; +``` + +### Parameters + +`ScrapeExecuteOptions` fields: + +| Field | Type | Description | +|---|---|---| +| `code` | `Option` | Code to run in the browser session. | +| `prompt` | `Option` | Natural-language instruction for the browser agent. | +| `language` | `Option` | Runtime: `Python`, `Node`, `Bash` (default `Node`). | +| `timeout` | `Option` | Execution timeout in seconds. | + +At least one of `code` or `prompt` must be provided. + +### Stop session + +`client.stop_interaction(job_id)` → `Result` + +## Notes + +- Deprecated aliases: `scrape_execute` → `interact`, `stop_interactive_browser`/`delete_scrape_browser` → `stop_interaction`. +- All methods are `async` and require a Tokio runtime. +- Fields use `snake_case` and are serialized to `camelCase` for the API automatically. +- `ScrapeOptions` implements `Default`, so use `..Default::default()` to fill unset fields. +- The `max_age` and `min_age` fields are in seconds in the Rust SDK (unlike milliseconds in other SDKs). + +## Source Of Truth + +- `firecrawl/apps/rust-sdk/Cargo.toml` +- `firecrawl/apps/rust-sdk/src/v2/client.rs` +- `firecrawl/apps/rust-sdk/src/v2/scrape.rs` +- `firecrawl/apps/rust-sdk/src/v2/search.rs` +- `firecrawl-docs/api-reference/v2-openapi.json`