diff --git a/.changeset/aisdk-allow-system-in-messages-act-extract-observe.md b/.changeset/aisdk-allow-system-in-messages-act-extract-observe.md new file mode 100644 index 000000000..1af6369a8 --- /dev/null +++ b/.changeset/aisdk-allow-system-in-messages-act-extract-observe.md @@ -0,0 +1,5 @@ +--- +"@browserbasehq/stagehand": patch +--- + +Remove the noisy AI SDK "system message in messages" warning from `act()`, `extract()`, and `observe()` (including when the agent's own tools call them internally). diff --git a/.changeset/config.json b/.changeset/config.json index b2875799c..52e56f343 100644 --- a/.changeset/config.json +++ b/.changeset/config.json @@ -3,6 +3,7 @@ "commit": false, "fixed": [], "linked": [], + "ignore": ["browse"], "baseBranch": "main", "updateInternalDependencies": "patch", "access": "public", diff --git a/.changeset/fifty-cats-sell.md b/.changeset/fifty-cats-sell.md deleted file mode 100644 index dfc981460..000000000 --- a/.changeset/fifty-cats-sell.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"@browserbasehq/stagehand": minor ---- - -extract links diff --git a/.changeset/floppy-experts-wash.md b/.changeset/floppy-experts-wash.md deleted file mode 100644 index 79a30b391..000000000 --- a/.changeset/floppy-experts-wash.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"@browserbasehq/stagehand": patch ---- - -remove unnecessary log diff --git a/.changeset/odysseysbench-eval-suite.md b/.changeset/odysseysbench-eval-suite.md new file mode 100644 index 000000000..6d9b5c3f3 --- /dev/null +++ b/.changeset/odysseysbench-eval-suite.md @@ -0,0 +1,5 @@ +--- +"@browserbasehq/stagehand-evals": minor +--- + +Add OdysseysBench as a supported agent benchmark in the evals CLI. OdysseysBench is a 200-task web-agent benchmark (45 easy / 46 medium / 109 hard); each task ships a weighted rubric that is baked into the verifier's `precomputed_rubric` format so process + outcome are scored against the published criteria. Run with `--eval-name agent/odysseysbench` (or the `external_agent_benchmarks` category); supports `EVAL_ODYSSEYSBENCH_LIMIT`, `EVAL_ODYSSEYSBENCH_SAMPLE`, `EVAL_ODYSSEYSBENCH_LEVEL`, and `EVAL_ODYSSEYSBENCH_IDS`. diff --git a/.changeset/solid-rice-admire.md b/.changeset/solid-rice-admire.md deleted file mode 100644 index 2f0291d02..000000000 --- a/.changeset/solid-rice-admire.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"@browserbasehq/stagehand": minor ---- - -Added Gemini 2.5 Flash to Google supported models diff --git a/.changeset/stagehand-server-v3-catchup-release.md b/.changeset/stagehand-server-v3-catchup-release.md new file mode 100644 index 000000000..3a46da2a9 --- /dev/null +++ b/.changeset/stagehand-server-v3-catchup-release.md @@ -0,0 +1,5 @@ +--- +"@browserbasehq/stagehand-server-v3": patch +--- + +Cut a new stagehand-server-v3 SEA binary release to catch up with recent core changes, including Gemini 3.5 Flash computer-use support. diff --git a/.changeset/vast-vans-crash.md b/.changeset/vast-vans-crash.md deleted file mode 100644 index 3fdc06f83..000000000 --- a/.changeset/vast-vans-crash.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"@browserbasehq/stagehand": patch ---- - -Fixes a redundant unnecessary log diff --git a/.changeset/wobbly-otters-deprecate.md b/.changeset/wobbly-otters-deprecate.md new file mode 100644 index 000000000..86e99bd16 --- /dev/null +++ b/.changeset/wobbly-otters-deprecate.md @@ -0,0 +1,5 @@ +--- +"browse": patch +--- + +Warn when auto-loading variables from `.env` and add `BROWSE_LOAD_DOTENV` to opt out ahead of a future release where this will be off by default. diff --git a/.cursorrules b/.cursorrules index fe68bca7b..fee9b45c2 100644 --- a/.cursorrules +++ b/.cursorrules @@ -1,140 +1,263 @@ # Stagehand Project -This is a project that uses Stagehand, which amplifies Playwright with `act`, `extract`, and `observe` added to the Page class. +This is a project that uses Stagehand V3, a browser automation framework with AI-powered `act`, `extract`, `observe`, and `agent` methods. -`Stagehand` is a class that provides config, a `StagehandPage` object via `stagehand.page`, and a `StagehandContext` object via `stagehand.context`. +The main class can be imported as `Stagehand` from `@browserbasehq/stagehand`. -`Page` is a class that extends the Playwright `Page` class and adds `act`, `extract`, and `observe` methods. -`Context` is a class that extends the Playwright `BrowserContext` class. +**Key Classes:** -Use the following rules to write code for this project. +- `Stagehand`: Main orchestrator class providing `act`, `extract`, `observe`, and `agent` methods +- `context`: A `V3Context` object that manages browser contexts and pages +- `page`: Individual page objects accessed via `stagehand.context.pages()[i]` or created with `stagehand.context.newPage()` -- To take an action on the page like "click the sign in button", use Stagehand `act` like this: +## Initialize ```typescript -await page.act("Click the sign in button"); +import { Stagehand } from "@browserbasehq/stagehand"; + +const stagehand = new Stagehand({ + env: "LOCAL", // or "BROWSERBASE" + verbose: 2, // 0, 1, or 2 + model: "openai/gpt-4.1-mini", // or any supported model +}); + +await stagehand.init(); + +// Access the browser context and pages +const page = stagehand.context.pages()[0]; +const context = stagehand.context; + +// Create new pages if needed +const page2 = await stagehand.context.newPage(); ``` -- To plan an instruction before taking an action, use Stagehand `observe` to get the action to execute. +## Act + +Actions are called on the `stagehand` instance (not the page). Use atomic, specific instructions: ```typescript -const [action] = await page.observe("Click the sign in button"); +// Act on the current active page +await stagehand.act("click the sign in button"); + +// Act on a specific page (when you need to target a page that isn't currently active) +await stagehand.act("click the sign in button", { page: page2 }); ``` -- The result of `observe` is an array of `ObserveResult` objects that can directly be used as params for `act` like this: +**Important:** Act instructions should be atomic and specific: - ```typescript - const [action] = await page.observe("Click the sign in button"); - await page.act(action); - ``` +- ✅ Good: "Click the sign in button" or "Type 'hello' into the search input" +- ❌ Bad: "Order me pizza" or "Type in the search bar and hit enter" (multi-step) -- When writing code that needs to extract data from the page, use Stagehand `extract`. Explicitly pass the following params by default: +### Observe + Act Pattern (Recommended) + +Cache the results of `observe` to avoid unexpected DOM changes: + +```typescript +const instruction = "Click the sign in button"; + +// Get candidate actions +const actions = await stagehand.observe(instruction); + +// Execute the first action +await stagehand.act(actions[0]); +``` + +To target a specific page: ```typescript -const { someValue } = await page.extract({ - instruction: the instruction to execute, - schema: z.object({ - someValue: z.string(), - }), // The schema to extract +const actions = await stagehand.observe("select blue as the favorite color", { + page: page2, }); +await stagehand.act(actions[0], { page: page2 }); ``` -## Initialize +## Extract + +Extract data from pages using natural language instructions. The `extract` method is called on the `stagehand` instance. + +### Basic Extraction (with schema) ```typescript -import { Stagehand } from "@browserbasehq/stagehand"; -import StagehandConfig from "./stagehand.config"; +import { z } from "zod"; + +// Extract with explicit schema +const data = await stagehand.extract( + "extract all apartment listings with prices and addresses", + z.object({ + listings: z.array( + z.object({ + price: z.string(), + address: z.string(), + }), + ), + }), +); -const stagehand = new Stagehand(StagehandConfig); -await stagehand.init(); +console.log(data.listings); +``` + +### Simple Extraction (without schema) + +```typescript +// Extract returns a default object with 'extraction' field +const result = await stagehand.extract("extract the sign in button text"); + +console.log(result); +// Output: { extraction: "Sign in" } -const page = stagehand.page; // Playwright Page with act, extract, and observe methods -const context = stagehand.context; // Playwright BrowserContext +// Or destructure directly +const { extraction } = await stagehand.extract( + "extract the sign in button text", +); +console.log(extraction); // "Sign in" ``` -## Act +### Targeted Extraction -You can cache the results of `observe` and use them as params for `act` like this: +Extract data from a specific element using a selector: ```typescript -const instruction = "Click the sign in button"; -const cachedAction = await getCache(instruction); +const reason = await stagehand.extract( + "extract the reason why script injection fails", + z.string(), + { selector: "/html/body/div[2]/div[3]/iframe/html/body/p[2]" }, +); +``` -if (cachedAction) { - await page.act(cachedAction); -} else { - try { - const results = await page.observe(instruction); - await setCache(instruction, results); - await page.act(results[0]); - } catch (error) { - await page.act(instruction); // If the action is not cached, execute the instruction directly - } -} +### URL Extraction + +When extracting links or URLs, use `z.string().url()`: + +```typescript +const { links } = await stagehand.extract( + "extract all navigation links", + z.object({ + links: z.array(z.string().url()), + }), +); ``` -Be sure to cache the results of `observe` and use them as params for `act` to avoid unexpected DOM changes. Using `act` without caching will result in more unpredictable behavior. +### Extracting from a Specific Page -Act `action` should be as atomic and specific as possible, i.e. "Click the sign in button" or "Type 'hello' into the search input". -AVOID actions that are more than one step, i.e. "Order me pizza" or "Type in the search bar and hit enter". +```typescript +// Extract from a specific page (when you need to target a page that isn't currently active) +const data = await stagehand.extract( + "extract the placeholder text on the name field", + { page: page2 }, +); +``` -## Extract +## Observe -If you are writing code that needs to extract data from the page, use Stagehand `extract`. +Plan actions before executing them. Returns an array of candidate actions: ```typescript -const signInButtonText = await page.extract("extract the sign in button text"); +// Get candidate actions on the current active page +const [action] = await stagehand.observe("Click the sign in button"); + +// Execute the action +await stagehand.act(action); ``` -You can also pass in params like an output schema in Zod, and a flag to use text extraction: +Observing on a specific page: ```typescript -const data = await page.extract({ - instruction: "extract the sign in button text", - schema: z.object({ - text: z.string(), - }), +// Target a specific page (when you need to target a page that isn't currently active) +const actions = await stagehand.observe("find the next page button", { + page: page2, }); +await stagehand.act(actions[0], { page: page2 }); ``` -`schema` is a Zod schema that describes the data you want to extract. To extract an array, make sure to pass in a single object that contains the array, as follows: +## Agent + +Use the `agent` method to autonomously execute complex, multi-step tasks. + +### Basic Agent Usage ```typescript -const data = await page.extract({ - instruction: "extract the text inside all buttons", - schema: z.object({ - text: z.array(z.string()), - }), - useTextExtract: true, // Set true for larger-scale extractions (multiple paragraphs), or set false for small extractions (name, birthday, etc) +const page = stagehand.context.pages()[0]; +await page.goto("https://www.google.com"); + +const agent = stagehand.agent({ + model: "google/gemini-2.0-flash", + executionModel: "google/gemini-2.0-flash", +}); + +const result = await agent.execute({ + instruction: "Search for the stock price of NVDA", + maxSteps: 20, }); + +console.log(result.message); ``` -## Agent +### Computer Use Agent (CUA) -Use the `agent` method to automonously execute larger tasks like "Get the stock price of NVDA" +For more advanced scenarios using computer-use models: ```typescript -// Navigate to a website -await stagehand.page.goto("https://www.google.com"); +const agent = stagehand.agent({ + mode: "cua", // Enable Computer Use Agent mode + model: "anthropic/claude-sonnet-4-20250514", + // or "google/gemini-2.5-computer-use-preview-10-2025" + systemPrompt: `You are a helpful assistant that can use a web browser. + Do not ask follow up questions, the user will trust your judgement.`, +}); + +await agent.execute({ + instruction: "Apply for a library card at the San Francisco Public Library", + maxSteps: 30, +}); +``` +### Agent with Custom Model Configuration + +```typescript const agent = stagehand.agent({ - // You can use either OpenAI or Anthropic - provider: "openai", - // The model to use (claude-3-7-sonnet-20250219 or claude-3-5-sonnet-20240620 for Anthropic) - model: "computer-use-preview", - - // Customize the system prompt - instructions: `You are a helpful assistant that can use a web browser. - Do not ask follow up questions, the user will trust your judgement.`, - - // Customize the API key - options: { - apiKey: process.env.OPENAI_API_KEY, + model: { + modelName: "google/gemini-2.5-computer-use-preview-10-2025", + apiKey: process.env.GEMINI_API_KEY, }, + systemPrompt: `You are a helpful assistant.`, }); +``` -// Execute the agent -await agent.execute( - "Apply for a library card at the San Francisco Public Library" -); +### Agent with Integrations (MCP/External Tools) + +```typescript +const agent = stagehand.agent({ + integrations: [`https://mcp.exa.ai/mcp?exaApiKey=${process.env.EXA_API_KEY}`], + systemPrompt: `You have access to the Exa search tool.`, +}); +``` + +## Advanced Features + +### DeepLocator (XPath Targeting) + +Target specific elements across shadow DOM and iframes: + +```typescript +await page + .deepLocator("/html/body/div[2]/div[3]/iframe/html/body/p") + .highlight({ + durationMs: 5000, + contentColor: { r: 255, g: 0, b: 0 }, + }); +``` + +### Multi-Page Workflows + +```typescript +const page1 = stagehand.context.pages()[0]; +await page1.goto("https://example.com"); + +const page2 = await stagehand.context.newPage(); +await page2.goto("https://example2.com"); + +// Act/extract/observe operate on the current active page by default +// Pass { page } option to target a specific page +await stagehand.act("click button", { page: page1 }); +await stagehand.extract("get title", { page: page2 }); ``` diff --git a/.env.example b/.env.example index f7b468d6f..eb08f00ee 100644 --- a/.env.example +++ b/.env.example @@ -6,7 +6,7 @@ BRAINTRUST_API_KEY="" ANTHROPIC_API_KEY="" HEADLESS=false ENABLE_CACHING=false -EVAL_MODELS="gpt-4o,claude-3-5-sonnet-latest" -EXPERIMENTAL_EVAL_MODELS="gpt-4o,claude-3-5-sonnet-latest,o1-mini,o1-preview" +EVAL_MODELS="gpt-4o,claude-sonnet-4-6" +EXPERIMENTAL_EVAL_MODELS="gpt-4o,claude-sonnet-4-6,o1-mini,o1-preview" EVAL_CATEGORIES="observe,act,combination,extract,experimental" -STAGEHAND_API_URL="http://localhost:80" +AGENT_EVAL_MAX_STEPS=50 \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 000000000..7cc4db16c --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,76 @@ +--- +name: Bug report +about: Detailed descriptions help us resolve faster +title: '' +labels: '' +assignees: '' + +--- + +**Before submitting an issue, please:** + +- [ ] Check the [documentation](https://docs.stagehand.dev/) for relevant information +- [ ] Search existing [issues](https://github.com/browserbase/stagehand/issues) to avoid duplicates + +## Environment Information + +Please provide the following information to help us reproduce and resolve your issue: + +**Stagehand:** + +- Language/SDK: [TypeScript, Python, MCP…] +- Stagehand version: [e.g., 1.0.0] + +**AI Provider:** + +- Provider: [e.g., OpenAI, Anthropic, Azure OpenAI] +- Model: [e.g., gpt-4o, claude-sonnet-4-6] + +## Issue Description + +``` +[Describe the current behavior here] + +``` + +### Steps to Reproduce + +1. +2. +3. + +### Minimal Reproduction Code + +```tsx +// Your minimal reproduction code here +import { Stagehand } from '@browserbase/stagehand'; + +const stagehand = new Stagehand({ + // IMPORTANT: include your stagehand config +}); + +// Steps that reproduce the issue + +``` + +### Error Messages / Log trace + +``` +[Paste error messages/logs here] + +``` + +### Screenshots / Videos + +``` +[Attach screenshots or videos here] + +``` + +### Related Issues + +Are there any related issues or PRs? + +- Related to: #[issue number] +- Duplicate of: #[issue number] +- Blocks: #[issue number] diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 000000000..75889eb82 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,23 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: '' +labels: '' +assignees: '' + +--- + +**Is your feature request related to a problem? Please describe.** +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +**Describe the solution you'd like** +A clear and concise description of what you want to happen. + +**Describe alternatives you've considered** +A clear and concise description of any alternative solutions or features you've considered. + +**Are you willing to contribute to implementing this feature or fix?** + +- [ ] Yes, I can submit a PR +- [ ] Yes, but I need guidance +- [ ] No, I cannot contribute at this time diff --git a/.github/actions/select-browserbase-region/action.yml b/.github/actions/select-browserbase-region/action.yml new file mode 100644 index 000000000..e5ccbd13e --- /dev/null +++ b/.github/actions/select-browserbase-region/action.yml @@ -0,0 +1,67 @@ +name: Select Browserbase region +description: Select a Browserbase region based on a weighted distribution. +inputs: + distribution: + description: Comma-separated region=weight list (e.g. us-west-2=40,us-east-1=20). + required: true +outputs: + region: + description: Selected region. + value: ${{ steps.select.outputs.region }} +runs: + using: composite + steps: + - id: select + shell: bash + run: | + dist="${{ inputs.distribution }}" + if [ -z "$dist" ]; then + echo "BROWSERBASE_REGION_DISTRIBUTION is empty" + exit 1 + fi + IFS=',' read -r -a entries <<< "$dist" + total=0 + regions=() + weights=() + for entry in "${entries[@]}"; do + region="${entry%%=*}" + weight="${entry#*=}" + region="$(printf '%s' "$region" | tr -d '[:space:]')" + weight="$(printf '%s' "$weight" | tr -d '[:space:]')" + if [ -z "$region" ] || [ -z "$weight" ]; then + echo "Invalid region distribution entry: $entry" + exit 1 + fi + if ! [[ "$region" =~ ^[A-Za-z0-9-]+$ ]]; then + echo "Invalid region value: $region" + exit 1 + fi + if ! [[ "$weight" =~ ^[0-9]+$ ]]; then + echo "Invalid weight for region $region: $weight" + exit 1 + fi + regions+=("$region") + weights+=("$weight") + total=$((total + weight)) + done + if [ "$total" -le 0 ]; then + echo "Invalid total weight: $total" + exit 1 + fi + roll=$((RANDOM % total)) + cumulative=0 + chosen="" + for i in "${!regions[@]}"; do + cumulative=$((cumulative + weights[i])) + if [ "$roll" -lt "$cumulative" ]; then + chosen="${regions[i]}" + break + fi + done + if [ -z "$chosen" ]; then + echo "Failed to choose Browserbase region" + exit 1 + fi + echo "Selected Browserbase region: $chosen" + echo "region=$chosen" >> "$GITHUB_OUTPUT" + echo "BROWSERBASE_REGION=$chosen" >> "$GITHUB_ENV" diff --git a/.github/actions/setup-node-pnpm-turbo/action.yml b/.github/actions/setup-node-pnpm-turbo/action.yml new file mode 100644 index 000000000..65bda9f79 --- /dev/null +++ b/.github/actions/setup-node-pnpm-turbo/action.yml @@ -0,0 +1,56 @@ +name: Setup Node, pnpm, and Turbo cache +description: Configure pnpm and Node.js with caching, restore Turbo cache, and install dependencies. +inputs: + node-version: + description: Node.js version to use. + required: false + default: "20.x" + use-prebuilt-artifacts: + description: Whether to download pre-built package from build artifacts. + required: false + default: "true" + restore-turbo-cache: + description: Whether to restore the local .turbo cache. + required: false + default: "true" + +runs: + using: composite + steps: + - uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4.3.0 + + - name: Set up Node.js + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: ${{ inputs.node-version }} + cache: 'pnpm' + cache-dependency-path: '**/pnpm-lock.yaml' + + - name: Restore Turbo cache + if: ${{ inputs.restore-turbo-cache == 'true' }} + uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: .turbo + key: ${{ runner.os }}-turbo-${{ hashFiles('pnpm-lock.yaml', 'pnpm-workspace.yaml', 'package.json', 'turbo.json') }}-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-turbo-${{ hashFiles('pnpm-lock.yaml', 'pnpm-workspace.yaml', 'package.json', 'turbo.json') }}- + + - name: Install dependencies + shell: bash + run: pnpm install --frozen-lockfile --prefer-offline + + - name: Download build artifacts + if: ${{ inputs.use-prebuilt-artifacts == 'true' }} + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: build-artifacts + path: . + merge-multiple: true + + - name: Prepare test output directories + shell: bash + run: | + mkdir -p "${GITHUB_WORKSPACE}/ctrf" + if [ -n "${NODE_V8_COVERAGE:-}" ]; then + mkdir -p "$NODE_V8_COVERAGE" + fi diff --git a/.github/actions/upload-ctrf-report/action.yml b/.github/actions/upload-ctrf-report/action.yml new file mode 100644 index 000000000..668c4e2f4 --- /dev/null +++ b/.github/actions/upload-ctrf-report/action.yml @@ -0,0 +1,34 @@ +name: Upload CTRF report +description: Upload CTRF report artifact. +inputs: + name: + description: Report path (used as artifact name when sanitized). + required: true + path: + description: Optional explicit path (defaults to name). + required: false + default: "" + +runs: + using: composite + steps: + - name: Normalize inputs + id: normalize + shell: bash + run: | + name="${{ inputs.name }}" + echo "name=${name//\//-}" >> "$GITHUB_OUTPUT" + if [ -n "${{ inputs.path }}" ]; then + echo "path=${{ inputs.path }}" >> "$GITHUB_OUTPUT" + else + echo "path=${{ inputs.name }}" >> "$GITHUB_OUTPUT" + fi + + - name: Upload CTRF report artifact + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: ${{ steps.normalize.outputs.name }} + # package.json anchors uploaded paths to the repository root. + path: | + package.json + ${{ steps.normalize.outputs.path }} diff --git a/.github/actions/upload-v8-coverage/action.yml b/.github/actions/upload-v8-coverage/action.yml new file mode 100644 index 000000000..cedb83c39 --- /dev/null +++ b/.github/actions/upload-v8-coverage/action.yml @@ -0,0 +1,34 @@ +name: Upload V8 coverage +description: Upload V8 coverage artifacts. +inputs: + name: + description: Artifact name. + required: true + path: + description: Coverage path to upload (defaults to name). + required: false + default: "" + +runs: + using: composite + steps: + - name: Normalize artifact name + id: normalize + shell: bash + run: | + name="${{ inputs.name }}" + echo "name=${name//\//-}" >> "$GITHUB_OUTPUT" + if [ -n "${{ inputs.path }}" ]; then + echo "path=${{ inputs.path }}" >> "$GITHUB_OUTPUT" + else + echo "path=${{ inputs.name }}" >> "$GITHUB_OUTPUT" + fi + + - name: Upload coverage artifact + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: ${{ steps.normalize.outputs.name }} + # package.json anchors uploaded paths to the repository root. + path: | + package.json + ${{ steps.normalize.outputs.path }} diff --git a/.github/actions/verify-chromium-launch/action.yml b/.github/actions/verify-chromium-launch/action.yml new file mode 100644 index 000000000..e8be8fdcb --- /dev/null +++ b/.github/actions/verify-chromium-launch/action.yml @@ -0,0 +1,223 @@ +name: Verify Chromium launch +description: Validate that Chromium can start, connect to CDP, and read the page title. +inputs: + chrome-path: + description: Path to Chromium/Chrome binary. + required: false + default: "/usr/bin/chromium" + max-attempts: + description: Number of launch attempts before failing. + required: false + default: "3" + timeout-ms: + description: Milliseconds to wait for DevTools and CDP per attempt. + required: false + default: "30000" +runs: + using: composite + steps: + - shell: bash + run: | + set -euo pipefail + max_attempts="${{ inputs.max-attempts }}" + attempt=1 + while [ "$attempt" -le "$max_attempts" ]; do + if [ -n "${{ inputs.chrome-path }}" ]; then + pkill -f "${{ inputs.chrome-path }}" >/dev/null 2>&1 || true + fi + if node - <<'NODE' + const { spawn } = require("node:child_process"); + const workspace = process.env.GITHUB_WORKSPACE; + if (workspace) { + process.chdir(workspace); + } + + const chrome = "${{ inputs.chrome-path }}"; + + const timeoutMs = Number("${{ inputs.timeout-ms }}"); + const wsPrefix = "DevTools listening on "; + const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); + + let proc; + let wsUrl; + + const waitForWsUrl = async () => { + const deadline = Date.now() + timeoutMs; + while (!wsUrl) { + if (Date.now() > deadline) { + throw new Error( + `❌ Chromium did not expose CDP WS URL within timeout (${timeoutMs}ms)`, + ); + } + await sleep(250); + } + return wsUrl; + }; + + const cleanup = () => { + if (proc && !proc.killed) { + proc.kill("SIGKILL"); + } + }; + + (async () => { + try { + const startTime = Date.now(); + const args = [ + '--ash-no-nudges', + '--block-new-web-contents', + '--deny-permission-prompts', + '--disable-breakpad', + '--disable-client-side-phishing-detection', + '--disable-component-update', + '--disable-components=AcceptCHFrame,OptimizationHints,ProcessPerSiteUpToMainFrameThreshold,InterestFeedContentSuggestions,CalculateNativeWinOcclusion,BackForwardCache,HeavyAdPrivacyMitigations,LazyFrameLoading,ImprovedCookieControls,PrivacySandboxSettings4,AutofillServerCommunication,CertificateTransparencyComponentUpdater,DestroyProfileOnBrowserClose,CrashReporting,OverscrollHistoryNavigation,InfiniteSessionRestore', + '--disable-datasaver-prompt', + '--disable-default-apps', + '--disable-desktop-notifications', + '--disable-domain-reliability', + '--disable-external-intent-requests', + '--disable-hang-monitor', + '--disable-infobars', + '--disable-notifications', + '--disable-popup-blocking', + '--disable-print-preview', + '--disable-prompt-on-repost', + '--disable-search-engine-choice-screen', + '--disable-session-crashed-bubble', + '--disable-speech-api', + '--disable-speech-synthesis-api', + '--hide-crash-restore-bubble', + '--metrics-recording-only', + '--no-default-browser-check', + '--no-first-run', + '--no-pings', + '--noerrdialogs', + '--safebrowsing-disable-auto-update', + '--silent-debugger-extension-api', + '--simulate-outdated-no-au="Tue, 31 Dec 2099 23:59:59 GMT"', + '--suppress-message-center-popups', + "--disable-background-networking", + "--disable-default-apps", + "--disable-dev-shm-usage", + "--disable-extensions", + "--disable-notifications", + "--disable-setuid-sandbox", + "--disable-site-isolation-trials", + "--disable-sync", + "--disable-web-security", + "--headless=new", + "--no-default-browser-check", + "--no-first-run", + "--no-sandbox", + "--no-zygote", + "--password-store=basic", + "--remote-debugging-port=0", + "--test-type=gpu", + "--use-mock-keychain", + "about:blank", + ]; + proc = spawn(chrome, args, { stdio: ["ignore", "pipe", "pipe"] }); + const lineBuffers = { stdout: "", stderr: "" }; + const onData = (stream) => (data) => { + const text = data.toString(); + if (stream === "stderr") { + process.stderr.write(text); + } else { + process.stdout.write(text); + } + lineBuffers[stream] += text; + const lines = lineBuffers[stream].split(/\r?\n/); + lineBuffers[stream] = lines.pop() ?? ""; + for (const line of lines) { + const idx = line.indexOf(wsPrefix); + if (idx === -1) continue; + const rest = line.slice(idx + wsPrefix.length).trim(); + const candidate = rest.split(/\s+/)[0]; + if ( + candidate.startsWith("ws://") || + candidate.startsWith("wss://") + ) { + wsUrl = candidate; + } + } + }; + proc.stdout.on("data", onData("stdout")); + proc.stderr.on("data", onData("stderr")); + + const url = await waitForWsUrl(); + const wsFoundMs = Date.now() - startTime; + const wsFoundSec = (wsFoundMs / 1000).toFixed(2); + const connectStart = Date.now(); + const path = require("node:path"); + const workspaceRoot = process.env.GITHUB_WORKSPACE || process.cwd(); + const playwrightPath = path.join( + workspaceRoot, + "packages/core/node_modules/playwright", + ); + console.log( + `✅ CDP Url found after ${wsFoundSec}s, connecting with playwright...`, + ); + const { chromium } = require(playwrightPath); + const browser = await chromium.connectOverCDP(url, { + timeout: timeoutMs, + }); + const context = browser.contexts()[0]; + if (!context) { + throw new Error("❌ No browser context available after CDP connect"); + } + const page = context.pages()[0]; + if (!page) { + throw new Error("❌ No page available after CDP connect"); + } + const remainingMs = timeoutMs - (Date.now() - connectStart); + if (remainingMs <= 0) { + throw new Error( + `❌ CDP connect + verify timed out after ${timeoutMs}ms`, + ); + } + const sum = await Promise.race([ + page.evaluate("1 + 1"), + new Promise((_, reject) => + setTimeout( + () => + reject( + new Error( + `❌ CDP connect + verify timed out after ${timeoutMs}ms`, + ), + ), + remainingMs, + ), + ), + ]); + if (sum !== 2) { + throw new Error(`❌ Unexpected eval result: ${sum}`); + } + const totalMs = Date.now() - startTime; + const connectMs = Date.now() - connectStart; + const totalSec = (totalMs / 1000).toFixed(2); + const connectSec = (connectMs / 1000).toFixed(2); + console.log( + `✅ Chromium launched in ${wsFoundSec}s and CDP connected in ${connectSec}s (total: ${totalSec}s)`, + ); + await browser.close(); + cleanup(); + process.exit(0); + } catch (err) { + cleanup(); + console.error(err instanceof Error ? err.message : String(err)); + process.exit(1); + } + })(); + NODE + then + if [ "$attempt" -gt 1 ]; then + echo "⚠️ Chromium launch succeeded after ${attempt} attempts; GitHub Actions runner may be constrained." + fi + exit 0 + fi + echo "⚠️ Chromium launch attempt ${attempt} failed." + attempt=$((attempt + 1)) + sleep 2 + done + echo "❌ Failed to launch Chromium before running Stagehand; GitHub Actions runner is likely overloaded." + exit 1 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 66fdc2dd8..2cceede26 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,4 +1,4 @@ -name: Evals +name: Tests on: pull_request: @@ -7,659 +7,967 @@ on: - synchronize - labeled - unlabeled + paths-ignore: + - "packages/docs/**" + workflow_dispatch: + inputs: + pull_request_number: + description: Internal mirrored pull request number + required: true + type: string + +permissions: + contents: read + actions: write + pull-requests: read env: - EVAL_MODELS: "gpt-4o,gpt-4o-mini,claude-3-5-sonnet-latest" - EVAL_CATEGORIES: "observe,act,combination,extract,text_extract,targeted_extract" + BROWSERBASE_FLOW_LOGS: "1" + LLM_MAX_MS: "15000" + EVAL_MODELS: "openai/gpt-4.1,google/gemini-2.5-flash,anthropic/claude-haiku-4-5" + EVAL_AGENT_MODELS: "openai/gpt-5.4-mini,anthropic/claude-haiku-4-5,google/gemini-3-flash-preview" + EVAL_CATEGORIES: "regression,observe,act,combination,extract,targeted_extract,agent" + EVAL_MAX_CONCURRENCY: 25 + EVAL_TRIAL_COUNT: 3 + LOCAL_SESSION_LIMIT_PER_E2E_TEST: 2 + BROWSERBASE_SESSION_LIMIT_PER_E2E_TEST: 3 + BROWSERBASE_REGION_DISTRIBUTION: "us-west-2=30,us-east-1=30,eu-central-1=20,ap-southeast-1=20" # percentage of load for each region when running e2e tests against prod + CHROME_PATH: /usr/bin/chromium # GitHub Actions runners ship with stable Chromium by default + BROWSERBASE_CDP_CONNECT_MAX_MS: "10000" + BROWSERBASE_SESSION_CREATE_MAX_MS: "60000" + PUPPETEER_SKIP_DOWNLOAD: "1" + PLAYWRIGHT_SKIP_DOWNLOAD: "1" + TURBO_TELEMETRY_DISABLED: "1" concurrency: - group: ${{ github.ref }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: + determine-changes: + runs-on: ubuntu-latest + outputs: + core: ${{ steps.filter.outputs.core }} + cli: ${{ steps.filter.outputs.cli }} + evals: ${{ steps.filter.outputs.evals }} + server: ${{ steps.filter.outputs.server }} + docs-only: ${{ steps.filter.outputs.docs-only }} + steps: + - name: Check out repository code + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + + - name: Log GitHub API rate limit + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + headers_file=$(mktemp) + body_file=$(mktemp) + curl -sSL \ + -D "$headers_file" \ + -o "$body_file" \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + -H "Authorization: Bearer $GITHUB_TOKEN" \ + https://api.github.com/rate_limit + cat "$headers_file" + echo "" + cat "$body_file" + remaining=$(jq -r '.rate.remaining' "$body_file") + if [ "$remaining" -eq 0 ]; then + reset_epoch=$(jq -r '.rate.reset' "$body_file") + reset_utc=$(date -u -d "@$reset_epoch" +"%Y-%m-%d %H:%M:%S") + reset_pacific=$(TZ=America/Los_Angeles date -d "@$reset_epoch" +"%Y-%m-%d %H:%M:%S %Z") + echo "Github API rate limited until: ${reset_pacific} (${reset_utc} UTC)" >> "$GITHUB_STEP_SUMMARY" + echo "GitHub API rate limit exhausted." + exit 1 + fi + + - uses: dorny/paths-filter@d1c1ffe0248fe513906c8e24db8ea791d46f8590 # v3.0.3 + id: filter + with: + filters: | + core: + - '.github/workflows/ci.yml' + - 'packages/core/**' + - 'package.json' + - 'pnpm-lock.yaml' + - 'turbo.json' + cli: + - 'packages/cli/**' + - 'packages/core/**' + - 'package.json' + - 'pnpm-lock.yaml' + evals: + - 'packages/evals/**' + - 'package.json' + - 'pnpm-lock.yaml' + server: + - 'packages/server-v3/**' + - 'packages/server-v4/**' + - 'packages/core/**' + - 'package.json' + - 'pnpm-lock.yaml' + - 'pnpm-workspace.yaml' + - '.github/workflows/ci.yml' + docs-only: + - '**/*.md' + - 'examples/**' + - '!packages/**/*.md' + + load-pr-context: + runs-on: ubuntu-latest + outputs: + pull_request_number: ${{ steps.load.outputs.pull_request_number }} + is_internal_head: ${{ steps.load.outputs.is_internal_head }} + skip_evals: ${{ steps.load.outputs.skip_evals }} + skip_regression_evals: ${{ steps.load.outputs.skip_regression_evals }} + label_combination: ${{ steps.load.outputs.label_combination }} + label_extract: ${{ steps.load.outputs.label_extract }} + label_act: ${{ steps.load.outputs.label_act }} + label_observe: ${{ steps.load.outputs.label_observe }} + label_targeted_extract: ${{ steps.load.outputs.label_targeted_extract }} + label_agent: ${{ steps.load.outputs.label_agent }} + steps: + - id: load + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const repoFullName = `${context.repo.owner}/${context.repo.repo}`; + let pr = context.payload.pull_request; + + if (!pr) { + const prNumber = Number('${{ github.event.inputs.pull_request_number }}'); + if (!prNumber) { + throw new Error('workflow_dispatch requires pull_request_number input'); + } + + const { data } = await github.rest.pulls.get({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: prNumber, + }); + pr = data; + } + + const labelNames = (pr.labels || []) + .map((label) => typeof label === 'string' ? label : label.name) + .filter(Boolean); + const hasLabel = (name) => labelNames.includes(name); + + core.setOutput('pull_request_number', String(pr.number)); + core.setOutput('is_internal_head', pr.head.repo?.full_name === repoFullName ? 'true' : 'false'); + core.setOutput('skip_evals', hasLabel('skip-evals') ? 'true' : 'false'); + core.setOutput('skip_regression_evals', hasLabel('skip-regression-evals') ? 'true' : 'false'); + core.setOutput('label_combination', hasLabel('combination') ? 'true' : 'false'); + core.setOutput('label_extract', hasLabel('extract') ? 'true' : 'false'); + core.setOutput('label_act', hasLabel('act') ? 'true' : 'false'); + core.setOutput('label_observe', hasLabel('observe') ? 'true' : 'false'); + core.setOutput('label_targeted_extract', hasLabel('targeted-extract') ? 'true' : 'false'); + core.setOutput('label_agent', hasLabel('agent') ? 'true' : 'false'); + determine-evals: + needs: [determine-changes, load-pr-context] runs-on: ubuntu-latest outputs: - run-combination: ${{ steps.check-labels.outputs.run-combination }} - run-extract: ${{ steps.check-labels.outputs.run-extract }} - run-act: ${{ steps.check-labels.outputs.run-act }} - run-observe: ${{ steps.check-labels.outputs.run-observe }} - run-text-extract: ${{ steps.check-labels.outputs.run-text-extract }} - run-targeted-extract: ${{ steps.check-labels.outputs.run-targeted-extract }} + skip-all-evals: ${{ steps.check-labels.outputs.skip-all-evals }} + eval-categories: ${{ steps.check-labels.outputs.eval-categories }} steps: - id: check-labels run: | - # Default to running all tests on main branch - if [[ "${{ github.ref }}" == "refs/heads/main" ]]; then - echo "Running all tests for main branch" - echo "run-combination=true" >> $GITHUB_OUTPUT - echo "run-extract=true" >> $GITHUB_OUTPUT - echo "run-act=true" >> $GITHUB_OUTPUT - echo "run-observe=true" >> $GITHUB_OUTPUT - echo "run-text-extract=true" >> $GITHUB_OUTPUT - echo "run-targeted-extract=true" >> $GITHUB_OUTPUT + categories=() + declare -A seen + add_category() { + local category="$1" + if [[ -z "${seen[$category]:-}" ]]; then + categories+=("$category") + seen["$category"]=1 + fi + } + + emit_categories() { + local json="[" + for category in "${categories[@]}"; do + json+="\"${category}\"," + done + json="${json%,}" + json+="]" + echo "eval-categories=$json" >> $GITHUB_OUTPUT + } + + # Check if skip-evals label is present + if [[ "${{ needs.load-pr-context.outputs.skip_evals }}" == "true" ]]; then + echo "skip-evals label found - skipping all evals" + echo "skip-all-evals=true" >> $GITHUB_OUTPUT + emit_categories exit 0 fi - # Check for specific labels - echo "run-combination=${{ contains(github.event.pull_request.labels.*.name, 'combination') }}" >> $GITHUB_OUTPUT - echo "run-extract=${{ contains(github.event.pull_request.labels.*.name, 'extract') }}" >> $GITHUB_OUTPUT - echo "run-act=${{ contains(github.event.pull_request.labels.*.name, 'act') }}" >> $GITHUB_OUTPUT - echo "run-observe=${{ contains(github.event.pull_request.labels.*.name, 'observe') }}" >> $GITHUB_OUTPUT - echo "run-text-extract=${{ contains(github.event.pull_request.labels.*.name, 'text-extract') }}" >> $GITHUB_OUTPUT - echo "run-targeted-extract=${{ contains(github.event.pull_request.labels.*.name, 'targeted-extract') }}" >> $GITHUB_OUTPUT + # Skip evals if only docs/examples changed + if [[ "${{ needs.determine-changes.outputs.docs-only }}" == "true" && "${{ needs.determine-changes.outputs.core }}" == "false" && "${{ needs.determine-changes.outputs.evals }}" == "false" ]]; then + echo "Only docs/examples changed - skipping evals" + echo "skip-all-evals=true" >> $GITHUB_OUTPUT + emit_categories + exit 0 + fi + + # Check for skip-regression-evals label + if [[ "${{ needs.load-pr-context.outputs.skip_regression_evals }}" == "true" ]]; then + echo "skip-regression-evals label found - regression evals will be skipped" + else + echo "Regression evals will run by default" + add_category "regression" + fi + # Check for specific labels + echo "skip-all-evals=false" >> $GITHUB_OUTPUT + if [[ "${{ needs.load-pr-context.outputs.label_combination }}" == "true" ]]; then + add_category "combination" + fi + if [[ "${{ needs.load-pr-context.outputs.label_extract }}" == "true" ]]; then + add_category "extract" + fi + if [[ "${{ needs.load-pr-context.outputs.label_act }}" == "true" ]]; then + add_category "act" + fi + if [[ "${{ needs.load-pr-context.outputs.label_observe }}" == "true" ]]; then + add_category "observe" + fi + if [[ "${{ needs.load-pr-context.outputs.label_targeted_extract }}" == "true" ]]; then + add_category "targeted_extract" + fi + if [[ "${{ needs.load-pr-context.outputs.label_agent }}" == "true" ]]; then + add_category "agent" + fi + emit_categories + run-lint: + name: Lint runs-on: ubuntu-latest + needs: [run-build] steps: - name: Check out repository code - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - - name: Set up Node.js - uses: actions/setup-node@v4 + - uses: ./.github/actions/setup-node-pnpm-turbo with: - node-version: "20" - - - name: Install dependencies - run: | - rm -rf node_modules - rm -f package-lock.json - npm install + use-prebuilt-artifacts: "true" + restore-turbo-cache: "false" + node-version: 20.x - name: Run Lint - run: npm run lint + run: pnpm exec turbo run lint + + cancel-after-lint-failure: + name: Cancel after lint failure + runs-on: ubuntu-latest + needs: [run-lint] + if: ${{ always() && needs.run-lint.result == 'failure' }} + continue-on-error: true + steps: + - name: Cancel workflow run + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + curl -sSfL -X POST \ + -H "Authorization: Bearer ${GITHUB_TOKEN}" \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "https://api.github.com/repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/cancel" run-build: + name: Build runs-on: ubuntu-latest steps: - name: Check out repository code - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - - name: Set up Node.js - uses: actions/setup-node@v4 + - uses: ./.github/actions/setup-node-pnpm-turbo with: - node-version: "20" - - - name: Install dependencies - run: | - rm -rf node_modules - rm -f package-lock.json - npm install + use-prebuilt-artifacts: "false" + node-version: 20.x - name: Run Build - run: npm run build - - run-e2e-tests: - needs: [run-lint, run-build] - runs-on: ubuntu-latest - timeout-minutes: 50 - env: - HEADLESS: true - steps: - - name: Check out repository code - uses: actions/checkout@v4 + run: pnpm exec turbo run build - - name: Set up Node.js - uses: actions/setup-node@v4 + - name: Save Turbo cache + if: always() + uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: - node-version: "20" + path: .turbo + key: ${{ runner.os }}-turbo-${{ hashFiles('pnpm-lock.yaml', 'pnpm-workspace.yaml', 'package.json', 'turbo.json') }}-${{ github.sha }} - - name: Install dependencies - run: | - rm -rf node_modules - rm -f package-lock.json - npm install + - name: Upload build artifacts + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: build-artifacts + include-hidden-files: true + # package.json is included to anchor artifact paths at repo root. + path: | + package.json + packages/core/dist/** + packages/core/lib/version.ts + packages/core/lib/dom/build/** + packages/core/lib/v3/dom/build/** + packages/cli/dist/** + packages/cli/oclif.manifest.json + packages/evals/dist/** + packages/server-v3/dist/** + packages/server-v3/openapi.v3.yaml + packages/server-v4/dist/** + packages/server-v4/openapi.v4.yaml + retention-days: 1 + + run-cli-tests: + name: CLI Tests (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + needs: [run-build, determine-changes] + if: needs.determine-changes.outputs.cli == 'true' + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest] + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + fetch-depth: 1 - - name: Install Playwright browsers - run: npm exec playwright install --with-deps + - uses: ./.github/actions/setup-node-pnpm-turbo + with: + use-prebuilt-artifacts: "true" + restore-turbo-cache: "false" + + # Runs the full CLI suite on each OS. POSIX-only tests opt out at the test + # site via the itPosix/describePosix helpers (tests/helpers/platform.ts), + # so coverage is on by default — a new test runs on every OS unless it + # explicitly skips. No test-name knowledge lives in this workflow. + - name: Run CLI Tests + run: pnpm exec turbo run test:cli --filter=browse + + run-evals-unit-tests: + name: Evals Unit Tests + runs-on: ubuntu-latest + needs: [run-build, determine-changes] + if: needs.determine-changes.outputs.evals == 'true' + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + fetch-depth: 1 - - name: Build Stagehand - run: npm run build + - uses: ./.github/actions/setup-node-pnpm-turbo + with: + use-prebuilt-artifacts: "true" + restore-turbo-cache: "false" - - name: Run E2E Tests (Deterministic Playwright) - run: npm run e2e + - name: Run Evals Unit Tests + run: pnpm --filter @browserbasehq/stagehand-evals run test:unit - run-e2e-local-tests: - needs: [run-lint, run-build] + discover-core-tests: runs-on: ubuntu-latest - timeout-minutes: 50 - env: - HEADLESS: true + needs: [determine-changes] + if: needs.determine-changes.outputs.core == 'true' + outputs: + core-tests: ${{ steps.set-matrix.outputs.core-tests }} + has-core-tests: ${{ steps.set-matrix.outputs.has-core-tests }} + steps: - - name: Check out repository code - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + fetch-depth: 1 - - name: Set up Node.js - uses: actions/setup-node@v4 + - uses: ./.github/actions/setup-node-pnpm-turbo with: - node-version: "20" + use-prebuilt-artifacts: "false" + restore-turbo-cache: "false" - - name: Install dependencies + - name: Discover core test files + id: set-matrix run: | - rm -rf node_modules - rm -f package-lock.json - npm install + core_json=$(pnpm --filter @browserbasehq/stagehand --silent run test:core -- --list) + echo "core-tests=$core_json" >> $GITHUB_OUTPUT - - name: Install Playwright browsers - run: npm exec playwright install --with-deps - - - name: Build Stagehand - run: npm run build + if [ "$core_json" = "[]" ]; then + echo "has-core-tests=false" >> $GITHUB_OUTPUT + else + echo "has-core-tests=true" >> $GITHUB_OUTPUT + fi - - name: Run local E2E Tests (Deterministic Playwright) - run: npm run e2e:local + echo "Found core tests: $core_json" - run-e2e-bb-tests: - needs: [run-lint, run-build] + core-unit-tests: + name: core/${{ matrix.test.name }} runs-on: ubuntu-latest - timeout-minutes: 50 - if: > - github.event_name == 'push' || - (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository) + needs: [run-build, discover-core-tests] + if: needs.discover-core-tests.outputs.has-core-tests == 'true' env: - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} - BROWSERBASE_API_KEY: ${{ secrets.BROWSERBASE_API_KEY }} - BROWSERBASE_PROJECT_ID: ${{ secrets.BROWSERBASE_PROJECT_ID }} - HEADLESS: true + STAGEHAND_BROWSER_TARGET: local + STAGEHAND_SERVER_TARGET: local + + strategy: + fail-fast: false + max-parallel: 100 + matrix: + test: ${{ fromJson(needs.discover-core-tests.outputs.core-tests) }} + steps: - - name: Check out repository code - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + fetch-depth: 1 - - name: Set up Node.js - uses: actions/setup-node@v4 + - uses: ./.github/actions/setup-node-pnpm-turbo with: - node-version: "20" + use-prebuilt-artifacts: "true" + restore-turbo-cache: "false" - - name: Install dependencies + - name: Run Vitest - ${{ matrix.test.name }} run: | - rm -rf node_modules - rm -f package-lock.json - npm install - - - name: Install Playwright browsers - run: npm exec playwright install --with-deps + pnpm exec turbo run test:core --only --filter=@browserbasehq/stagehand -- "${{ matrix.test.path }}" - - name: Build Stagehand - run: npm run build + - uses: ./.github/actions/upload-ctrf-report + if: always() + with: + name: ctrf/core-unit/${{ matrix.test.name }}.json - - name: Run E2E Tests (browserbase) - run: npm run e2e:bb + - uses: ./.github/actions/upload-v8-coverage + if: always() + with: + name: coverage/core-unit/${{ matrix.test.name }} - run-regression-evals: - needs: - [run-e2e-bb-tests, run-e2e-tests, run-e2e-local-tests, determine-evals] + discover-server-tests: runs-on: ubuntu-latest - timeout-minutes: 9 + needs: [determine-changes] + if: needs.determine-changes.outputs.server == 'true' outputs: - regression_score: ${{ steps.set-regression-score.outputs.regression_score }} - env: - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} - BRAINTRUST_API_KEY: ${{ secrets.BRAINTRUST_API_KEY }} - BROWSERBASE_API_KEY: ${{ secrets.BROWSERBASE_API_KEY }} - BROWSERBASE_PROJECT_ID: ${{ secrets.BROWSERBASE_PROJECT_ID }} - HEADLESS: true - EVAL_ENV: browserbase + integration-tests: ${{ steps.set-matrix.outputs.integration-tests }} + has-integration-tests: ${{ steps.set-matrix.outputs.has-integration-tests }} + steps: - - name: Check out repository code - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + fetch-depth: 1 - - name: Set up Node.js - uses: actions/setup-node@v4 + - uses: ./.github/actions/setup-node-pnpm-turbo with: - node-version: "20" + use-prebuilt-artifacts: "false" + restore-turbo-cache: "false" - - name: Install dependencies + - name: Discover server test files + id: set-matrix run: | - rm -rf node_modules - rm -f package-lock.json - npm install - - - name: Build Stagehand - run: npm run build - - - name: Install Playwright browsers - run: npm exec playwright install --with-deps + int_json=$(pnpm --filter @browserbasehq/stagehand-server-v3 --silent run test:server -- --list integration) + echo "integration-tests=$int_json" >> $GITHUB_OUTPUT - - name: Run Regression Evals - run: npm run evals category regression trials=2 concurrency=20 env=BROWSERBASE - - - name: Log Regression Evals Performance - run: | - experimentName=$(jq -r '.experimentName' eval-summary.json) - echo "View results at https://www.braintrust.dev/app/Browserbase/p/stagehand/experiments/${experimentName}" - if [ -f eval-summary.json ]; then - regression_score=$(jq '.categories.regression' eval-summary.json) - echo "Regression category score: $regression_score%" - if (( $(echo "$regression_score < 90" | bc -l) )); then - echo "Regression category score is below 90%. Failing CI." - exit 1 - fi + if [ "$int_json" = "[]" ]; then + echo "has-integration-tests=false" >> $GITHUB_OUTPUT else - echo "Eval summary not found for regression category. Failing CI." - exit 1 + echo "has-integration-tests=true" >> $GITHUB_OUTPUT fi - run-combination-evals: - needs: [run-regression-evals, determine-evals] + echo "Found server integration tests: $int_json" + + build-server-sea: + name: Build SEA binary (tests, v3) + uses: ./.github/workflows/stagehand-server-v3-sea-build.yml + needs: [run-build] + with: + matrix: | + [ + {"os":"ubuntu-latest","platform":"linux","arch":"x64","binary_name":"stagehand-server-v3-linux-x64","include_sourcemaps":false}, + {"os":"ubuntu-24.04-arm","platform":"linux","arch":"arm64","binary_name":"stagehand-server-v3-linux-arm64","include_sourcemaps":false}, + {"os":"macos-15","platform":"darwin","arch":"arm64","binary_name":"stagehand-server-v3-darwin-arm64","include_sourcemaps":false}, + {"os":"macos-15-intel","platform":"darwin","arch":"x64","binary_name":"stagehand-server-v3-darwin-x64","include_sourcemaps":false}, + {"os":"windows-latest","platform":"win32","arch":"x64","binary_name":"stagehand-server-v3-win32-x64.exe","include_sourcemaps":false}, + {"os":"windows-11-arm","platform":"win32","arch":"arm64","binary_name":"stagehand-server-v3-win32-arm64.exe","include_sourcemaps":false}, + {"os":"ubuntu-latest","platform":"linux","arch":"x64","binary_name":"stagehand-server-v3-linux-x64-sourcemap","include_sourcemaps":true} + ] + use-prebuilt-artifacts: "true" + restore-turbo-cache: "false" + node-version: "20.x" + upload-only-binary: stagehand-server-v3-linux-x64-sourcemap + + server-integration-tests: + name: server/v3/integration/${{ matrix.test.name }} runs-on: ubuntu-latest - timeout-minutes: 40 + needs: [build-server-sea, discover-server-tests, run-build] + if: needs.discover-server-tests.outputs.has-integration-tests == 'true' + + strategy: + fail-fast: false + matrix: + test: ${{ fromJson(needs.discover-server-tests.outputs.integration-tests) }} + env: + BB_ENV: local + STAGEHAND_API_URL: http://stagehand-api.localhost:3106 + STAGEHAND_BROWSER_TARGET: local + STAGEHAND_SERVER_TARGET: sea OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} - BRAINTRUST_API_KEY: ${{ secrets.BRAINTRUST_API_KEY }} + # Used only for testing /start with env: BROWSERBASE remote browser BROWSERBASE_API_KEY: ${{ secrets.BROWSERBASE_API_KEY }} BROWSERBASE_PROJECT_ID: ${{ secrets.BROWSERBASE_PROJECT_ID }} - HEADLESS: true - EVAL_ENV: browserbase + steps: - - name: Check out repository code - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + fetch-depth: 1 + + - uses: ./.github/actions/setup-node-pnpm-turbo + with: + use-prebuilt-artifacts: "true" + restore-turbo-cache: "false" - - name: Check for 'combination' label - id: label-check + - name: Download SEA binary + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: stagehand-server-v3-linux-x64-sourcemap + path: . + + - name: Ensure SEA binary is present and executable + shell: bash run: | - if [ "${{ needs.determine-evals.outputs.run-combination }}" != "true" ]; then - echo "has_label=false" >> $GITHUB_OUTPUT - echo "No label for COMBINATION. Exiting with success." - else - echo "has_label=true" >> $GITHUB_OUTPUT - fi + set -euo pipefail + test -f packages/server-v3/dist/sea/stagehand-server-v3-linux-x64-sourcemap + chmod +x packages/server-v3/dist/sea/stagehand-server-v3-linux-x64-sourcemap + + - name: Set up Chrome + id: setup-chrome-server + uses: browser-actions/setup-chrome@2e1d749697dd1612b833dba4a722266286fbefcd # v2.1.2 + with: + chrome-version: stable + install-dependencies: true + + - name: Export Chrome path + run: | + echo "CHROME_PATH=${{ steps.setup-chrome-server.outputs.chrome-path }}" >> "$GITHUB_ENV" - - name: Set up Node.js - if: needs.determine-evals.outputs.run-combination == 'true' - uses: actions/setup-node@v4 + - uses: ./.github/actions/verify-chromium-launch with: - node-version: "20" + chrome-path: ${{ env.CHROME_PATH }} - - name: Install dependencies - if: needs.determine-evals.outputs.run-combination == 'true' + - name: Run server integration test - ${{ matrix.test.name }} + env: + SEA_BINARY_NAME: stagehand-server-v3-linux-x64-sourcemap run: | - rm -rf node_modules - rm -f package-lock.json - npm install + pnpm exec turbo run test:server --only --filter=@browserbasehq/stagehand-server-v3 -- "${{ matrix.test.path }}" + + - uses: ./.github/actions/upload-ctrf-report + if: always() + with: + name: ctrf/server-v3-integration/${{ matrix.test.name }}.json - - name: Build Stagehand - if: needs.determine-evals.outputs.run-combination == 'true' - run: npm run build + - uses: ./.github/actions/upload-v8-coverage + if: always() + with: + name: coverage/server-v3-integration/${{ matrix.test.name }} - - name: Install Playwright browsers - if: needs.determine-evals.outputs.run-combination == 'true' - run: npm exec playwright install --with-deps + discover-e2e-tests: + runs-on: ubuntu-latest + needs: [determine-changes] + if: needs.determine-changes.outputs.core == 'true' + outputs: + e2e-tests: ${{ steps.set-matrix.outputs.e2e-tests }} + has-e2e-tests: ${{ steps.set-matrix.outputs.has-e2e-tests }} - - name: Run Combination Evals - if: needs.determine-evals.outputs.run-combination == 'true' - run: npm run evals category combination + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + fetch-depth: 1 - - name: Log Combination Evals Performance - if: needs.determine-evals.outputs.run-combination == 'true' + - uses: ./.github/actions/setup-node-pnpm-turbo + with: + use-prebuilt-artifacts: "false" + restore-turbo-cache: "false" + + - name: Discover e2e test files + id: set-matrix run: | - experimentName=$(jq -r '.experimentName' eval-summary.json) - echo "View results at https://www.braintrust.dev/app/Browserbase/p/stagehand/experiments/${experimentName}" - if [ -f eval-summary.json ]; then - combination_score=$(jq '.categories.combination' eval-summary.json) - echo "Combination category score: $combination_score%" - exit 0 + e2e_json=$(pnpm --filter @browserbasehq/stagehand --silent run test:e2e -- --list) + echo "e2e-tests=$e2e_json" >> $GITHUB_OUTPUT + + if [ "$e2e_json" = "[]" ]; then + echo "has-e2e-tests=false" >> $GITHUB_OUTPUT else - echo "Eval summary not found for combination category. Failing CI." - exit 1 + echo "has-e2e-tests=true" >> $GITHUB_OUTPUT fi - run-act-evals: - needs: [run-combination-evals, determine-evals] + echo "Found e2e tests: $e2e_json" + + run-e2e-local-tests: + name: e2e/local/${{ matrix.test.name }} + needs: [run-build, discover-e2e-tests, load-pr-context] runs-on: ubuntu-latest - timeout-minutes: 25 + timeout-minutes: 50 + if: > + needs.discover-e2e-tests.outputs.has-e2e-tests == 'true' && + needs.load-pr-context.outputs.is_internal_head == 'true' env: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} - BRAINTRUST_API_KEY: ${{ secrets.BRAINTRUST_API_KEY }} + GOOGLE_GENERATIVE_AI_API_KEY: ${{ secrets.GOOGLE_GENERATIVE_AI_API_KEY }} BROWSERBASE_API_KEY: ${{ secrets.BROWSERBASE_API_KEY }} BROWSERBASE_PROJECT_ID: ${{ secrets.BROWSERBASE_PROJECT_ID }} HEADLESS: true - EVAL_ENV: browserbase + STAGEHAND_BROWSER_TARGET: local + STAGEHAND_SERVER_TARGET: local + strategy: + fail-fast: false + max-parallel: 20 + matrix: + test: ${{ fromJson(needs.discover-e2e-tests.outputs.e2e-tests) }} steps: - name: Check out repository code - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - - name: Check for 'act' label - id: label-check - run: | - if [ "${{ needs.determine-evals.outputs.run-act }}" != "true" ]; then - echo "has_label=false" >> $GITHUB_OUTPUT - echo "No label for ACT. Exiting with success." - else - echo "has_label=true" >> $GITHUB_OUTPUT - fi + - uses: ./.github/actions/setup-node-pnpm-turbo + with: + use-prebuilt-artifacts: "true" + restore-turbo-cache: "false" - - name: Set up Node.js - if: needs.determine-evals.outputs.run-act == 'true' - uses: actions/setup-node@v4 + - name: Set up Chrome + id: setup-chrome-local-e2e + uses: browser-actions/setup-chrome@2e1d749697dd1612b833dba4a722266286fbefcd # v2.1.2 with: - node-version: "20" + chrome-version: stable + install-dependencies: true - - name: Install dependencies - if: needs.determine-evals.outputs.run-act == 'true' + - name: Export Chrome path run: | - rm -rf node_modules - rm -f package-lock.json - npm install + echo "CHROME_PATH=${{ steps.setup-chrome-local-e2e.outputs.chrome-path }}" >> "$GITHUB_ENV" - - name: Build Stagehand - if: needs.determine-evals.outputs.run-act == 'true' - run: npm run build + - uses: ./.github/actions/verify-chromium-launch + with: + chrome-path: ${{ env.CHROME_PATH }} - - name: Install Playwright browsers - if: needs.determine-evals.outputs.run-act == 'true' - run: npm exec playwright install --with-deps + - name: Run local E2E Tests - ${{ matrix.test.name }} + run: | + pnpm exec turbo run test:e2e --only --filter=@browserbasehq/stagehand -- "${{ matrix.test.path }}" - - name: Run Act Evals - if: needs.determine-evals.outputs.run-act == 'true' - run: npm run evals category act + - uses: ./.github/actions/upload-ctrf-report + if: always() + with: + name: ctrf/e2e-local/${{ matrix.test.name }}.json - - name: Log Act Evals Performance - if: needs.determine-evals.outputs.run-act == 'true' - run: | - experimentName=$(jq -r '.experimentName' eval-summary.json) - echo "View results at https://www.braintrust.dev/app/Browserbase/p/stagehand/experiments/${experimentName}" - if [ -f eval-summary.json ]; then - act_score=$(jq '.categories.act' eval-summary.json) - echo "Act category score: $act_score%" - if (( $(echo "$act_score < 80" | bc -l) )); then - echo "Act category score is below 80%. Failing CI." - exit 1 - fi - else - echo "Eval summary not found for act category. Failing CI." - exit 1 - fi + - uses: ./.github/actions/upload-v8-coverage + if: always() + with: + name: coverage/e2e-local/${{ matrix.test.name }} - run-extract-evals: - needs: [run-act-evals, determine-evals] + run-e2e-bb-tests: + name: e2e/bb/${{ matrix.test.name }} + needs: [run-build, discover-e2e-tests, load-pr-context] runs-on: ubuntu-latest timeout-minutes: 50 + if: > + needs.discover-e2e-tests.outputs.has-e2e-tests == 'true' && + needs.load-pr-context.outputs.is_internal_head == 'true' env: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} - BRAINTRUST_API_KEY: ${{ secrets.BRAINTRUST_API_KEY }} + GOOGLE_GENERATIVE_AI_API_KEY: ${{ secrets.GOOGLE_GENERATIVE_AI_API_KEY }} BROWSERBASE_API_KEY: ${{ secrets.BROWSERBASE_API_KEY }} BROWSERBASE_PROJECT_ID: ${{ secrets.BROWSERBASE_PROJECT_ID }} HEADLESS: true - EVAL_ENV: browserbase + STAGEHAND_BROWSER_TARGET: browserbase + STAGEHAND_SERVER_TARGET: local + strategy: + fail-fast: false + max-parallel: 100 + matrix: + test: ${{ fromJson(needs.discover-e2e-tests.outputs.e2e-tests) }} steps: - name: Check out repository code - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - - name: Check for 'extract' label - id: label-check - run: | - if [ "${{ needs.determine-evals.outputs.run-extract }}" != "true" ]; then - echo "has_label=false" >> $GITHUB_OUTPUT - echo "No label for EXTRACT. Exiting with success." - else - echo "has_label=true" >> $GITHUB_OUTPUT - fi + - uses: ./.github/actions/setup-node-pnpm-turbo + with: + use-prebuilt-artifacts: "true" + restore-turbo-cache: "false" - - name: Set up Node.js - if: needs.determine-evals.outputs.run-extract == 'true' - uses: actions/setup-node@v4 + - name: Select Browserbase region + uses: ./.github/actions/select-browserbase-region with: - node-version: "20" + distribution: ${{ env.BROWSERBASE_REGION_DISTRIBUTION }} - - name: Install dependencies - if: needs.determine-evals.outputs.run-extract == 'true' + - name: Run E2E Tests (browserbase) - ${{ matrix.test.name }} run: | - rm -rf node_modules - rm -f package-lock.json - npm install - - - name: Build Stagehand - if: needs.determine-evals.outputs.run-extract == 'true' - run: npm run build - - - name: Install Playwright browsers - if: needs.determine-evals.outputs.run-extract == 'true' - run: npm exec playwright install --with-deps - - # 1. Run extract category with domExtract - - name: Run Extract Evals (domExtract) - if: needs.determine-evals.outputs.run-extract == 'true' - run: npm run evals category extract -- --extract-method=domExtract - - - name: Save Extract Dom Results - if: needs.determine-evals.outputs.run-extract == 'true' - run: mv eval-summary.json eval-summary-extract-dom.json - - # 2. Then run extract category with textExtract - - name: Run Extract Evals (textExtract) - if: needs.determine-evals.outputs.run-extract == 'true' - run: npm run evals category extract -- --extract-method=textExtract - - - name: Save Extract Text Results - if: needs.determine-evals.outputs.run-extract == 'true' - run: mv eval-summary.json eval-summary-extract-text.json - - # 3. Log and Compare Extract Evals Performance - - name: Log and Compare Extract Evals Performance - if: needs.determine-evals.outputs.run-extract == 'true' - run: | - experimentNameDom=$(jq -r '.experimentName' eval-summary-extract-dom.json) - dom_score=$(jq '.categories.extract' eval-summary-extract-dom.json) - echo "DomExtract Extract category score: $dom_score%" - echo "View domExtract results: https://www.braintrust.dev/app/Browserbase/p/stagehand/experiments/${experimentNameDom}" - - experimentNameText=$(jq -r '.experimentName' eval-summary-extract-text.json) - text_score=$(jq '.categories.extract' eval-summary-extract-text.json) - echo "TextExtract Extract category score: $text_score%" - echo "View textExtract results: https://www.braintrust.dev/app/Browserbase/p/stagehand/experiments/${experimentNameText}" - - # If domExtract <80% fail CI - if (( $(echo "$dom_score < 80" | bc -l) )); then - echo "DomExtract extract category score is below 80%. Failing CI." - exit 1 - fi + pnpm exec turbo run test:e2e --only --filter=@browserbasehq/stagehand -- "${{ matrix.test.path }}" - run-text-extract-evals: - needs: [run-extract-evals, determine-evals] + - uses: ./.github/actions/upload-ctrf-report + if: always() + with: + name: ctrf/e2e-bb/${{ matrix.test.name }}.json + + - uses: ./.github/actions/upload-v8-coverage + if: always() + with: + name: coverage/e2e-bb/${{ matrix.test.name }} + + run-evals: + name: evals/${{ matrix.category }} + needs: [run-build, determine-evals, run-e2e-bb-tests] + if: >- + ${{ + always() && + needs.run-build.result == 'success' && + needs.determine-evals.result == 'success' && + needs.run-e2e-bb-tests.result != 'failure' && + needs.run-e2e-bb-tests.result != 'cancelled' && + needs.determine-evals.outputs.skip-all-evals != 'true' && + needs.determine-evals.outputs.eval-categories != '[]' + }} runs-on: ubuntu-latest - timeout-minutes: 120 + timeout-minutes: 90 + strategy: + fail-fast: false + matrix: + category: ${{ fromJson(needs.determine-evals.outputs.eval-categories) }} env: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + GOOGLE_GENERATIVE_AI_API_KEY: ${{ secrets.GOOGLE_GENERATIVE_AI_API_KEY }} BRAINTRUST_API_KEY: ${{ secrets.BRAINTRUST_API_KEY }} BROWSERBASE_API_KEY: ${{ secrets.BROWSERBASE_API_KEY }} BROWSERBASE_PROJECT_ID: ${{ secrets.BROWSERBASE_PROJECT_ID }} - HEADLESS: true - EVAL_ENV: browserbase + STAGEHAND_BROWSER_TARGET: browserbase + STAGEHAND_SERVER_TARGET: local steps: - name: Check out repository code - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - - name: Check for 'text-extract' label - id: label-check - run: | - if [ "${{ needs.determine-evals.outputs.run-text-extract }}" != "true" ]; then - echo "has_label=false" >> $GITHUB_OUTPUT - echo "No label for TEXT-EXTRACT. Exiting with success." - else - echo "has_label=true" >> $GITHUB_OUTPUT - fi + - uses: ./.github/actions/setup-node-pnpm-turbo + with: + use-prebuilt-artifacts: "true" + restore-turbo-cache: "false" - - name: Set up Node.js - if: needs.determine-evals.outputs.run-text-extract == 'true' - uses: actions/setup-node@v4 + - name: Select Browserbase region + uses: ./.github/actions/select-browserbase-region with: - node-version: "20" + distribution: ${{ env.BROWSERBASE_REGION_DISTRIBUTION }} - - name: Install dependencies - if: needs.determine-evals.outputs.run-text-extract == 'true' + - name: Run Evals - ${{ matrix.category }} + id: run-evals + env: + NODE_V8_COVERAGE: coverage/evals/${{ matrix.category }} run: | - rm -rf node_modules - rm -f package-lock.json - npm install - - - name: Install Playwright browsers - if: needs.determine-evals.outputs.run-text-extract == 'true' - run: npm exec playwright install --with-deps + set +e + pnpm exec turbo run test:evals --only --filter=@browserbasehq/stagehand-evals -- run "${{ matrix.category }}" -e browserbase -t "${EVAL_TRIAL_COUNT}" -c "${EVAL_MAX_CONCURRENCY}" + eval_status=$? + set -e - - name: Build Stagehand - if: needs.determine-evals.outputs.run-text-extract == 'true' - run: npm run build - - - name: Run text_extract Evals (textExtract) - if: needs.determine-evals.outputs.run-text-extract == 'true' - run: npm run evals category text_extract -- --extract-method=textExtract + if [ -f eval-summary.json ]; then + { + echo "summary_text<> "$GITHUB_OUTPUT" + fi - - name: Save text_extract Results - if: needs.determine-evals.outputs.run-text-extract == 'true' - run: mv eval-summary.json eval-summary-text_extract-text.json + exit "$eval_status" - - name: Log text_extract Evals Performance - if: needs.determine-evals.outputs.run-text-extract == 'true' + - name: Log Evals Performance - ${{ matrix.category }} + env: + EVAL_STDOUT_SUMMARY: ${{ steps.run-evals.outputs.summary_text }} run: | - experimentNameText=$(jq -r '.experimentName' eval-summary-text_extract-text.json) - text_score=$(jq '.categories.text_extract' eval-summary-text_extract-text.json) - echo "TextExtract text_extract category score: $text_score%" - echo "View textExtract results: https://www.braintrust.dev/app/Browserbase/p/stagehand/experiments/${experimentNameText}" - - # If text_score <80% fail CI - if (( $(echo "$text_score < 80" | bc -l) )); then - echo "textExtract text_extract category score is below 80%. Failing CI." - exit 1 + if [ -n "${EVAL_STDOUT_SUMMARY:-}" ]; then + echo "### Evals Summary (${{ matrix.category }})" >> "$GITHUB_STEP_SUMMARY" + echo '```' >> "$GITHUB_STEP_SUMMARY" + printf '%s\n' "$EVAL_STDOUT_SUMMARY" >> "$GITHUB_STEP_SUMMARY" + echo '```' >> "$GITHUB_STEP_SUMMARY" fi - run-observe-evals: - needs: [run-text-extract-evals, determine-evals] - runs-on: ubuntu-latest - timeout-minutes: 60 - env: - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} - BRAINTRUST_API_KEY: ${{ secrets.BRAINTRUST_API_KEY }} - BROWSERBASE_API_KEY: ${{ secrets.BROWSERBASE_API_KEY }} - BROWSERBASE_PROJECT_ID: ${{ secrets.BROWSERBASE_PROJECT_ID }} - HEADLESS: true - EVAL_ENV: browserbase - steps: - - name: Check out repository code - uses: actions/checkout@v4 + if [ ! -f eval-summary.json ]; then + echo "Eval summary not found for ${{ matrix.category }} category. Failing CI." + exit 1 + fi - - name: Check for 'observe' label - id: label-check - run: | - if [ "${{ needs.determine-evals.outputs.run-observe }}" != "true" ]; then - echo "has_label=false" >> $GITHUB_OUTPUT - echo "No label for OBSERVE. Exiting with success." + experimentUrl=$(jq -r '.experimentUrl // empty' eval-summary.json) + if [ -n "$experimentUrl" ]; then + echo "View results at ${experimentUrl}" else - echo "has_label=true" >> $GITHUB_OUTPUT + experimentName=$(jq -r '.experimentName' eval-summary.json) + echo "View results at https://www.braintrust.dev/app/Browserbase/p/stagehand/experiments/${experimentName}" + fi + + primary_score=$(jq -r '(.scores["Exact match"].score // .scores["Pass"].score // empty)' eval-summary.json) + if [ -z "$primary_score" ]; then + echo "Braintrust primary score not found in eval summary. Failing CI." + exit 1 + fi + + primary_score_percent=$(jq -rn --arg score "$primary_score" '$score | tonumber * 100') + printf '${{ matrix.category }} Braintrust score: %.2f%%\n' "$primary_score_percent" + if (( $(echo "$primary_score < 0.8" | bc -l) )); then + echo "${{ matrix.category }} Braintrust score is below 80%. Failing CI." + exit 1 fi - - name: Set up Node.js - if: needs.determine-evals.outputs.run-observe == 'true' - uses: actions/setup-node@v4 + - uses: ./.github/actions/upload-ctrf-report + if: always() with: - node-version: "20" + name: ctrf/evals/${{ matrix.category }}.json - - name: Install dependencies - if: needs.determine-evals.outputs.run-observe == 'true' - run: | - rm -rf node_modules - rm -f package-lock.json - npm install + - uses: ./.github/actions/upload-v8-coverage + if: always() + with: + name: coverage/evals/${{ matrix.category }} - - name: Install Playwright browsers - if: needs.determine-evals.outputs.run-observe == 'true' - run: npm exec playwright install --with-deps + merge-coverage: + name: Code Coverage Report + runs-on: ubuntu-latest + needs: + - core-unit-tests + - run-e2e-local-tests + - run-e2e-bb-tests + - run-evals + - server-integration-tests + - load-pr-context + # if: always() + if: false + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + fetch-depth: 1 - - name: Build Stagehand - if: needs.determine-evals.outputs.run-observe == 'true' - run: npm run build + - uses: ./.github/actions/setup-node-pnpm-turbo + with: + use-prebuilt-artifacts: "true" + restore-turbo-cache: "false" - - name: Run Observe Evals - if: needs.determine-evals.outputs.run-observe == 'true' - run: npm run evals category observe + - name: Download V8 coverage artifacts + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + continue-on-error: true + with: + pattern: coverage-* + path: . + merge-multiple: true - - name: Log Observe Evals Performance - if: needs.determine-evals.outputs.run-observe == 'true' - run: | - experimentName=$(jq -r '.experimentName' eval-summary.json) - echo "View results at https://www.braintrust.dev/app/Browserbase/p/stagehand/experiments/${experimentName}" - if [ -f eval-summary.json ]; then - observe_score=$(jq '.categories.observe' eval-summary.json) - echo "Observe category score: $observe_score%" - if (( $(echo "$observe_score < 80" | bc -l) )); then - echo "Observe category score is below 80%. Failing CI." - exit 1 - fi - else - echo "Eval summary not found for observe category. Failing CI." - exit 1 - fi + - name: Download CTRF artifacts + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + continue-on-error: true + with: + pattern: ctrf-* + path: . + merge-multiple: true - run-targeted-extract-evals: - needs: [run-observe-evals, determine-evals] - runs-on: ubuntu-latest - timeout-minutes: 60 - env: - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} - BRAINTRUST_API_KEY: ${{ secrets.BRAINTRUST_API_KEY }} - BROWSERBASE_API_KEY: ${{ secrets.BROWSERBASE_API_KEY }} - BROWSERBASE_PROJECT_ID: ${{ secrets.BROWSERBASE_PROJECT_ID }} - HEADLESS: true - EVAL_ENV: browserbase - steps: - - name: Check out repository code - uses: actions/checkout@v4 + - name: Generate merged coverage report + run: | + pnpm run coverage:merge - - name: Check for 'targeted-extract' label - id: label-check + - name: Upload merged coverage report + if: always() + id: upload-coverage-artifact + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: coverage-merged + # package.json is included to anchor artifact paths at repo root. + path: | + package.json + coverage/merged + + - name: Add coverage summary to job summary + if: always() + shell: bash run: | - if [ "${{ needs.determine-evals.outputs.run-targeted-extract }}" != "true" ]; then - echo "has_label=false" >> $GITHUB_OUTPUT - echo "No label for TARGETED-EXTRACT. Exiting with success." + echo "### Code Coverage" >> "$GITHUB_STEP_SUMMARY" + echo "" >> "$GITHUB_STEP_SUMMARY" + if [ -f coverage/merged/coverage-summary.txt ]; then + echo '```' >> "$GITHUB_STEP_SUMMARY" + cat coverage/merged/coverage-summary.txt >> "$GITHUB_STEP_SUMMARY" + echo '```' >> "$GITHUB_STEP_SUMMARY" else - echo "has_label=true" >> $GITHUB_OUTPUT + echo "Coverage summary not available." >> "$GITHUB_STEP_SUMMARY" + fi + if [ -n "${{ steps.upload-coverage-artifact.outputs.artifact-url }}" ]; then + echo "" >> "$GITHUB_STEP_SUMMARY" + echo "[Download full HTML coverage report](${{ steps.upload-coverage-artifact.outputs.artifact-url }})" >> "$GITHUB_STEP_SUMMARY" fi - - name: Set up Node.js - if: needs.determine-evals.outputs.run-targeted-extract == 'true' - uses: actions/setup-node@v4 + - name: Publish merged CTRF report + if: always() + uses: ctrf-io/github-test-reporter@0f299074936c32ccaab5be5230511f6b2b9080aa # v1.0.28 with: - node-version: "20" - - - name: Install dependencies - if: needs.determine-evals.outputs.run-targeted-extract == 'true' + report-path: './ctrf/**/*.json' + summary: true + summary-report: false + summary-delta-report: true + test-report: false + failed-report: false + insights-report: true + flaky-rate-report: true + fail-rate-report: true + slowest-report: true + previous-results-report: true + fetch-previous-results: true + baseline: 1 + previous-results-max: 1 + max-workflow-runs-to-check: 5 + max-previous-runs-to-fetch: 1 + upload-artifact: true + artifact-name: ctrf-report-merged + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Compute coverage status metrics + if: always() + id: coverage-status + shell: bash run: | - rm -rf node_modules - rm -f package-lock.json - npm install - - - name: Install Playwright browsers - if: needs.determine-evals.outputs.run-targeted-extract == 'true' - run: npm exec playwright install --with-deps - - - name: Build Stagehand - if: needs.determine-evals.outputs.run-targeted-extract == 'true' - run: npm run build - - - name: Run targeted extract Evals - if: needs.determine-evals.outputs.run-targeted-extract == 'true' - run: npm run evals category targeted_extract -- --extract-method=textExtract - - - name: Log targeted extract Evals Performance - if: needs.determine-evals.outputs.run-targeted-extract == 'true' + set -euo pipefail + shopt -s globstar nullglob + tests_failed=0 + ctrf_files=(ctrf/**/*.json) + if [ "${#ctrf_files[@]}" -gt 0 ]; then + tests_failed=$(jq -s '[.[].results.summary.failed // 0] | add' "${ctrf_files[@]}") + fi + total_coverage=0 + if [ -f coverage/merged/coverage-summary.txt ]; then + total_coverage=$(awk '/^Lines/ {gsub(/%/,"",$3); print $3}' coverage/merged/coverage-summary.txt) + fi + echo "tests_failed=${tests_failed}" >> "$GITHUB_OUTPUT" + echo "total_coverage=${total_coverage}" >> "$GITHUB_OUTPUT" + + - name: Set coverage status + if: always() + continue-on-error: true + shell: bash + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + RUN_ID: ${{ github.run_id }} + PULL_NUMBER: ${{ needs.load-pr-context.outputs.pull_request_number }} + TESTS_FAILED: ${{ steps.coverage-status.outputs.tests_failed }} + TOTAL_COVERAGE: ${{ steps.coverage-status.outputs.total_coverage }} run: | - experimentName=$(jq -r '.experimentName' eval-summary.json) - echo "View results at https://www.braintrust.dev/app/Browserbase/p/stagehand/experiments/${experimentName}" - if [ -f eval-summary.json ]; then - targeted_extract_score=$(jq '.categories.targeted_extract' eval-summary.json) - echo "Targeted extract category score: $targeted_extract_score%" - if (( $(echo "$targeted_extract_score < 80" | bc -l) )); then - echo "Targeted extract score is below 80%. Failing CI." - exit 1 - fi + set -euo pipefail + repo="${GITHUB_REPOSITORY}" + sha="${GITHUB_SHA}" + tests_failed="${TESTS_FAILED:-0}" + total_coverage="${TOTAL_COVERAGE:-0}" + state="success" + if [ -n "${PULL_NUMBER:-}" ]; then + target_url="https://github.com/${repo}/pull/${PULL_NUMBER}/checks?check_run_id=${RUN_ID}" else - echo "Eval summary not found for targeted_extract category. Failing CI." - exit 1 + target_url="https://github.com/${repo}/actions/runs/${RUN_ID}" fi + description="non-blocking report: ${tests_failed} tests failed. ${total_coverage}% coverage" + payload=$(jq -n \ + --arg state "$state" \ + --arg target_url "$target_url" \ + --arg description "$description" \ + --arg context "Measured coverage" \ + '{state: $state, target_url: $target_url, description: $description, context: $context}') + curl -sSfL -X POST \ + -H "Authorization: Bearer ${GITHUB_TOKEN}" \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "https://api.github.com/repos/${repo}/statuses/${sha}" \ + -d "$payload" diff --git a/.github/workflows/claude.yml b/.github/workflows/claude.yml new file mode 100644 index 000000000..22ad31bc6 --- /dev/null +++ b/.github/workflows/claude.yml @@ -0,0 +1,72 @@ +name: Claude Code + +on: + issue_comment: + types: [created] + pull_request_review_comment: + types: [created] + issues: + types: [opened, assigned] + pull_request_review: + types: [submitted] + +env: + BROWSERBASE_FLOW_LOGS: "1" + +jobs: + claude: + if: | + (github.event_name == 'issue_comment' && contains(github.event.comment.body, '@claude')) || + (github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '@claude')) || + (github.event_name == 'pull_request_review' && contains(github.event.review.body, '@claude')) || + (github.event_name == 'issues' && (contains(github.event.issue.body, '@claude') || contains(github.event.issue.title, '@claude'))) + runs-on: ubuntu-latest + permissions: + contents: write + pull-requests: write + issues: write + id-token: write + actions: write # Required for Claude to read CI results on PRs / rerun actions that failed + steps: + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + fetch-depth: 1 + + - uses: ./.github/actions/setup-node-pnpm-turbo + with: + use-prebuilt-artifacts: "false" + restore-turbo-cache: "false" + node-version: 20.x + + - name: Run Build + run: | + pnpm exec turbo run build + + - name: Run Claude Code + id: claude + uses: anthropics/claude-code-action@86eb26bf0139bdd75acd15ea5f00f45ee0a284c2 # v1.0.122 + with: + anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }} + + # This is an optional setting that allows Claude to read CI results on PRs + additional_permissions: | + actions: read + + track_progress: true + + # Optional: Give a custom prompt to Claude. If this is not specified, Claude will perform the instructions specified in the comment that tagged it. + prompt: 'Make sure "turbo run lint" and "turbo run build" pass before pushing and make sure to check present CI status for the branch and fix any easy failures. Prefer using the Github MCP tools over bash for Github operations, fall back to Bash(gh) for anything not supported by the MCP tools.' + + branch_prefix: 'claude-' + # Optional: Add claude_args to customize behavior and configuration + # See https://github.com/anthropics/claude-code-action/blob/main/docs/usage.md + # or https://code.claude.com/docs/en/cli-reference for available options + claude_args: | + --allowed-tools mcp__github_inline_comment__create_inline_comment,Bash,View,Glob,GlobTool,GrepTool,Grep,BatchTool,WebSearch,LS,Edit,MultiEdit,Write,Read + + +# consider adding in the future: +# - https://github.com/anthropics/claude-code-action/blob/main/examples/test-failure-analysis.yml +# - https://github.com/anthropics/claude-code-action/blob/main/examples/ci-failure-auto-fix.yml +# - https://github.com/anthropics/claude-code-action/blob/main/examples/issue-deduplication.yml diff --git a/.github/workflows/external-contributor-pr-approval-handoff.yml b/.github/workflows/external-contributor-pr-approval-handoff.yml new file mode 100644 index 000000000..ecf238942 --- /dev/null +++ b/.github/workflows/external-contributor-pr-approval-handoff.yml @@ -0,0 +1,43 @@ +name: External Contributor PR Approval Handoff + +on: + pull_request_review: + types: + - submitted + +permissions: + contents: read + pull-requests: read + +jobs: + capture-approved-review: + runs-on: ubuntu-latest + steps: + - name: Write approval handoff payload + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const fs = require('fs'); + const pr = context.payload.pull_request; + const review = context.payload.review; + const shouldClaim = + review.state === 'approved' && + pr.head.repo.full_name !== context.payload.repository.full_name; + + const payload = { + shouldClaim, + prNumber: pr.number, + reviewer: review.user?.login || '', + reviewId: review.id, + approvedSha: review.commit_id || pr.head.sha, + }; + + fs.writeFileSync('approval-handoff.json', JSON.stringify(payload)); + + - name: Upload approval handoff artifact + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: approved-review + path: approval-handoff.json + retention-days: 1 diff --git a/.github/workflows/external-contributor-pr.yml b/.github/workflows/external-contributor-pr.yml new file mode 100644 index 000000000..57fc01b91 --- /dev/null +++ b/.github/workflows/external-contributor-pr.yml @@ -0,0 +1,619 @@ +name: External Contributor PR + +on: + pull_request_target: + types: + - opened + - reopened + - synchronize + - closed + workflow_run: + workflows: + - External Contributor PR Approval Handoff + types: + - completed + +permissions: + actions: write + contents: write + pull-requests: write + issues: write + +env: + ECPR_LIB: | + (() => { + const LABELS = [ + { name: 'external-contributor', color: '8b949e', description: 'Tracks PRs mirrored from external contributor forks.' }, + { name: 'external-contributor:awaiting-approval', color: 'd29922', description: 'Waiting for a stagehand team member to approve the latest external commit.' }, + { name: 'external-contributor:mirrored', color: '1f6feb', description: 'An internal mirrored PR currently exists for this external contributor PR.' }, + { name: 'external-contributor:stale', color: 'db6d28', description: 'The mirrored PR is stale and waiting for a fresh approval to refresh.' }, + { name: 'external-contributor:completed', color: '2da44e', description: 'The mirrored PR has been merged and the external contributor flow is complete.' }, + ]; + const MANAGED_LABELS = new Set(LABELS.map((label) => label.name)); + const MANAGED_COMMENT_AUTHOR = 'github-actions[bot]'; + const CLAIM_RE = //; + const OWNED_RE = //; + const NOTICE_MARKER = ''; + const NOTICE_LINES = [ + 'This PR is from an external contributor and must be approved by a stagehand team member with write access before CI can run.', + 'Approving the latest commit mirrors it into an internal PR owned by the approver.', + 'If new commits are pushed later, the internal PR stays open but is marked stale until someone approves the latest external commit and refreshes it.', + ]; + + async function ensureLabels(github, context) { + for (const label of LABELS) { + try { + await github.rest.issues.getLabel({ owner: context.repo.owner, repo: context.repo.repo, name: label.name }); + } catch (error) { + if (error.status !== 404) throw error; + try { + await github.rest.issues.createLabel({ + owner: context.repo.owner, + repo: context.repo.repo, + name: label.name, + color: label.color, + description: label.description, + }); + } catch (createError) { + if (createError.status !== 422) throw createError; + } + } + } + } + + async function listComments(github, context, issueNumber) { + return github.paginate(github.rest.issues.listComments, { + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: issueNumber, + per_page: 100, + }); + } + + function isManagedComment(comment) { + return comment.user?.login === MANAGED_COMMENT_AUTHOR; + } + + function defaultManagedBranch(prNumber) { + return `external-contributor-pr-${prNumber}`; + } + + function sanitizeManagedBranch(prNumber, branch) { + const fallback = defaultManagedBranch(prNumber); + if (!branch) return fallback; + const allowed = new RegExp(`^external-contributor-pr-${prNumber}(?:-[A-Za-z0-9._-]+)?$`); + return allowed.test(branch) ? branch : fallback; + } + + async function upsertComment(github, context, issueNumber, marker, lines) { + const comments = await listComments(github, context, issueNumber); + const body = [marker, ...lines].join('\n'); + const existing = comments.find((comment) => isManagedComment(comment) && comment.body?.includes(marker)); + if (!existing) { + await github.rest.issues.createComment({ owner: context.repo.owner, repo: context.repo.repo, issue_number: issueNumber, body }); + return; + } + if (existing.body !== body) { + await github.rest.issues.updateComment({ owner: context.repo.owner, repo: context.repo.repo, comment_id: existing.id, body }); + } + } + + async function syncLabels(github, context, issueNumber, desiredLabels) { + const { data: issue } = await github.rest.issues.get({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: issueNumber, + }); + const existingNames = issue.labels.map((label) => typeof label === 'string' ? label : label.name).filter(Boolean); + const preserved = existingNames.filter((label) => !MANAGED_LABELS.has(label)); + await github.rest.issues.setLabels({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: issueNumber, + labels: [...preserved, ...desiredLabels], + }); + } + + async function findLatestClaim(github, context, issueNumber) { + const comments = await listComments(github, context, issueNumber); + return [...comments] + .reverse() + .map((comment) => { + if (!isManagedComment(comment)) return null; + const match = comment.body?.match(CLAIM_RE); + if (!match) return null; + const sourcePrNumber = issueNumber; + return { + ownedPrNumber: Number(match[1]), + sourceSha: match[2], + claimer: match[3], + branch: sanitizeManagedBranch(sourcePrNumber, match[4]), + }; + }) + .find(Boolean); + } + + async function externalLifecycle({ github, context }) { + const pr = context.payload.pull_request; + await ensureLabels(github, context); + + if (context.payload.action === 'opened' || context.payload.action === 'reopened') { + await upsertComment(github, context, pr.number, NOTICE_MARKER, NOTICE_LINES); + const latestClaim = await findLatestClaim(github, context, pr.number); + if (context.payload.action === 'reopened' && latestClaim && latestClaim.sourceSha === pr.head.sha) { + const { data: ownedPr } = await github.rest.pulls.get({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: latestClaim.ownedPrNumber, + }); + if (ownedPr.state === 'open') { + await syncLabels(github, context, pr.number, ['external-contributor', 'external-contributor:mirrored']); + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: pr.number, + body: `This external contributor PR is already mirrored to ${ownedPr.html_url}. Closing it again so discussion stays on the internal PR until fresh commits require another approval.`, + }); + await github.rest.pulls.update({ owner: context.repo.owner, repo: context.repo.repo, pull_number: pr.number, state: 'closed' }); + return; + } + } + await syncLabels(github, context, pr.number, ['external-contributor', 'external-contributor:awaiting-approval']); + return; + } + + const latestClaim = await findLatestClaim(github, context, pr.number); + if (!latestClaim || latestClaim.sourceSha === pr.head.sha) return; + + const { data: ownedPr } = await github.rest.pulls.get({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: latestClaim.ownedPrNumber, + }); + if (ownedPr.state !== 'open') return; + + await syncLabels(github, context, pr.number, ['external-contributor', 'external-contributor:awaiting-approval']); + await syncLabels(github, context, ownedPr.number, ['external-contributor', 'external-contributor:stale']); + await upsertComment(github, context, ownedPr.number, '', [ + `This mirrored PR is stale because the original external contributor PR #${pr.number} received new commits (\`${latestClaim.sourceSha}\` -> \`${pr.head.sha}\`).`, + `Original PR: ${pr.html_url}`, + '', + 'Approve the latest external commit to refresh this same internal PR in place.', + ]); + if (pr.state === 'closed') { + await github.rest.pulls.update({ owner: context.repo.owner, repo: context.repo.repo, pull_number: pr.number, state: 'open' }); + } + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: ownedPr.number, + body: `New commits landed on external contributor PR #${pr.number} (\`${latestClaim.sourceSha}\` -> \`${pr.head.sha}\`). This mirrored PR stays open but is now stale until the latest external commit is approved and copied over.`, + }); + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: pr.number, + body: `New commits were pushed to this external contributor PR (\`${latestClaim.sourceSha}\` -> \`${pr.head.sha}\`). The mirrored PR ${ownedPr.html_url} remains open but is marked stale. A stagehand team member with write access must approve the latest commit to refresh that internal PR.`, + }); + } + + async function prepareClaim({ github, context, core, artifactPath }) { + const fs = require('fs'); + const handoff = JSON.parse(fs.readFileSync(artifactPath, 'utf8')); + core.setOutput('should-claim', 'false'); + if (!handoff.shouldClaim || !handoff.prNumber || !handoff.reviewer || !handoff.approvedSha) return; + + const { data: pr } = await github.rest.pulls.get({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: Number(handoff.prNumber), + }); + if (pr.head.repo.full_name === context.payload.repository.full_name || pr.state !== 'open') return; + + const { data: permission } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: context.repo.owner, + repo: context.repo.repo, + username: handoff.reviewer, + }); + if (!new Set(['admin', 'maintain', 'write']).has(permission.permission)) { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: pr.number, + body: `@${handoff.reviewer} submitted an approving review, but only stagehand team members with write access can claim external contributor PRs. A maintainer with write access must approve the latest commit to proceed.`, + }); + return; + } + if (pr.head.sha !== handoff.approvedSha) return; + + const latestClaim = await findLatestClaim(github, context, pr.number); + const branch = sanitizeManagedBranch(pr.number, latestClaim?.branch); + const title = `[Claimed #${pr.number}] ${pr.title}`; + const body = [ + `Mirrored from external contributor PR #${pr.number} after approval by @${handoff.reviewer}.`, + '', + `Original author: @${pr.user.login}`, + `Original PR: ${pr.html_url}`, + `Approved source head SHA: \`${pr.head.sha}\``, + '', + `@${pr.user.login}, please continue any follow-up discussion on this mirrored PR. When the external PR gets new commits, this same internal PR will be marked stale until the latest external commit is approved and refreshed here.`, + '', + '## Original description', + pr.body?.trim() || '_No description provided._', + '', + ``, + ].join('\n'); + + const { data: ownedPrs } = await github.rest.pulls.list({ + owner: context.repo.owner, + repo: context.repo.repo, + state: 'all', + head: `${context.repo.owner}:${branch}`, + base: 'main', + per_page: 100, + }); + + core.setOutput('should-claim', 'true'); + core.setOutput('claimer', handoff.reviewer); + core.setOutput('pr-number', String(pr.number)); + core.setOutput('source-sha', pr.head.sha); + core.setOutput('previous-source-sha', latestClaim?.sourceSha || ''); + core.setOutput('branch', branch); + core.setOutput('title', title); + core.setOutput('body', body); + core.setOutput('owned-pr-number', ownedPrs[0] ? String(ownedPrs[0].number) : ''); + core.setOutput('owned-pr-merged', ownedPrs[0]?.merged_at ? 'true' : 'false'); + } + + async function finalizeClaim({ github, context, input }) { + await ensureLabels(github, context); + const { + prNumber, + sourceSha, + branch, + claimer, + title, + body, + existingNumber, + existingMerged, + refreshStatus, + refreshReason, + } = input; + + if (refreshStatus !== 'updated') { + if (existingNumber) { + await syncLabels(github, context, Number(existingNumber), ['external-contributor', 'external-contributor:stale']); + await upsertComment(github, context, Number(existingNumber), '', [ + `This mirrored PR could not be refreshed automatically after approval by @${claimer}.`, + '', + `Refresh reason: \`${refreshReason || 'unknown'}\``, + 'Resolve the branch manually, then keep using this same mirrored PR.', + ]); + } + await syncLabels(github, context, prNumber, ['external-contributor', 'external-contributor:awaiting-approval']); + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: prNumber, + body: `The latest approval by @${claimer} could not refresh the mirrored PR automatically (${refreshReason || 'unknown reason'}). The external PR stays open, and the mirrored PR should be updated manually before work continues.`, + }); + return { claimed: false, ownedPrNumber: existingNumber || '' }; + } + + let ownedPr; + if (existingNumber && !existingMerged) { + const { data } = await github.rest.pulls.update({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: Number(existingNumber), + title, + body, + base: 'main', + state: 'open', + }); + ownedPr = data; + } else { + const { data } = await github.rest.pulls.create({ + owner: context.repo.owner, + repo: context.repo.repo, + title, + body, + head: branch, + base: 'main', + }); + ownedPr = data; + } + + await github.rest.issues.addAssignees({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: ownedPr.number, + assignees: [claimer], + }); + await syncLabels(github, context, prNumber, ['external-contributor', 'external-contributor:mirrored']); + await syncLabels(github, context, ownedPr.number, ['external-contributor', 'external-contributor:mirrored']); + await upsertComment(github, context, ownedPr.number, '', [ + `This mirrored PR tracks external contributor PR #${prNumber} at source SHA \`${sourceSha}\`, approved by @${claimer}.`, + `Original PR: ${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/pull/${prNumber}`, + '', + 'When the external PR gets new commits, this same internal PR will be refreshed in place after the latest external commit is approved.', + ]); + + const marker = ``; + const comments = await listComments(github, context, prNumber); + if (!comments.some((comment) => comment.body?.includes(marker))) { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: prNumber, + body: [marker, `This PR was approved by @${claimer} and mirrored to ${ownedPr.html_url}. All further discussion should happen on that PR.`].join('\n'), + }); + } + + const { data: externalPr } = await github.rest.pulls.get({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: prNumber, + }); + if (externalPr.state !== 'closed') { + await github.rest.pulls.update({ owner: context.repo.owner, repo: context.repo.repo, pull_number: prNumber, state: 'closed' }); + } + + return { claimed: true, ownedPrNumber: String(ownedPr.number) }; + } + + async function syncOwnedPr({ github, context }) { + const pr = context.payload.pull_request; + const match = pr.body?.match(OWNED_RE); + if (!match) return; + + const sourcePrNumber = Number(match[1]); + const sourceSha = match[2]; + await ensureLabels(github, context); + + const { data: externalPr } = await github.rest.pulls.get({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: sourcePrNumber, + }); + + if (context.payload.action === 'reopened') { + await syncLabels(github, context, pr.number, ['external-contributor', 'external-contributor:mirrored']); + await syncLabels(github, context, sourcePrNumber, ['external-contributor', 'external-contributor:mirrored']); + if (externalPr.state !== 'closed') { + await github.rest.pulls.update({ owner: context.repo.owner, repo: context.repo.repo, pull_number: sourcePrNumber, state: 'closed' }); + } + return; + } + + if (pr.merged) { + await syncLabels(github, context, pr.number, ['external-contributor', 'external-contributor:completed']); + await syncLabels(github, context, sourcePrNumber, ['external-contributor', 'external-contributor:completed']); + await upsertComment(github, context, pr.number, '', [ + `This mirrored PR has been merged into \`main\`. The original external PR ${externalPr.html_url} is now completed.`, + ]); + await upsertComment(github, context, sourcePrNumber, ``, [ + `The mirrored PR ${pr.html_url} has been merged into \`main\`. This original external contributor PR will stay closed as completed.`, + ]); + return; + } + + await syncLabels(github, context, pr.number, ['external-contributor', 'external-contributor:stale']); + await syncLabels(github, context, sourcePrNumber, ['external-contributor', 'external-contributor:awaiting-approval']); + if (externalPr.head.sha !== sourceSha) { + await upsertComment(github, context, pr.number, '', [ + `This mirrored PR is stale because the original external PR ${externalPr.html_url} now points at a different source SHA.`, + 'Approve the latest external commit to refresh this same internal PR.', + ]); + return; + } + + if (externalPr.state === 'closed') { + await github.rest.pulls.update({ owner: context.repo.owner, repo: context.repo.repo, pull_number: sourcePrNumber, state: 'open' }); + } + await upsertComment(github, context, sourcePrNumber, ``, [ + `The mirrored PR ${pr.html_url} was closed without merge. This original PR has been reopened and is awaiting a fresh approving review from a stagehand team member with write access.`, + ]); + await upsertComment(github, context, pr.number, '', [ + `This mirrored PR was closed without merge. The original external PR ${externalPr.html_url} has been reopened and relabeled as awaiting approval.`, + ]); + } + + return { externalLifecycle, prepareClaim, finalizeClaim, syncOwnedPr }; + })() + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.event.workflow_run.id }} + cancel-in-progress: false + +jobs: + manage-external-pr: + if: github.event_name == 'pull_request_target' && github.event.pull_request.head.repo.full_name != github.repository + runs-on: ubuntu-latest + steps: + - name: Sync external PR lifecycle + if: github.event.action == 'opened' || github.event.action == 'reopened' || github.event.action == 'synchronize' + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const lib = eval(process.env.ECPR_LIB); + await lib.externalLifecycle({ github, context }); + + claim-approved-pr: + if: github.event_name == 'workflow_run' && github.event.workflow_run.conclusion == 'success' + runs-on: ubuntu-latest + steps: + - name: Download approval handoff artifact + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: approved-review + path: approval-handoff + github-token: ${{ secrets.GITHUB_TOKEN }} + repository: ${{ github.repository }} + run-id: ${{ github.event.workflow_run.id }} + + - name: Prepare approved claim + id: prepare-claim + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const lib = eval(process.env.ECPR_LIB); + await lib.prepareClaim({ github, context, core, artifactPath: 'approval-handoff/approval-handoff.json' }); + + - name: Checkout repository for branch operations + if: steps.prepare-claim.outputs.should-claim == 'true' + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + fetch-depth: 0 + persist-credentials: true + + - name: Refresh internal branch + if: steps.prepare-claim.outputs.should-claim == 'true' + id: refresh-branch + continue-on-error: true + env: + INTERNAL_BRANCH: ${{ steps.prepare-claim.outputs.branch }} + PR_NUMBER: ${{ steps.prepare-claim.outputs.pr-number }} + PREVIOUS_SOURCE_SHA: ${{ steps.prepare-claim.outputs.previous-source-sha }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -uo pipefail + + refresh_status="conflict" + refresh_reason="unknown" + + write_outputs() { + echo "refresh-status=${refresh_status}" >> "$GITHUB_OUTPUT" + if [ -n "${refresh_reason}" ]; then + echo "reason=${refresh_reason}" >> "$GITHUB_OUTPUT" + fi + } + + trap write_outputs EXIT + + if ! git config user.name "github-actions[bot]"; then + refresh_reason="git-config-failed" + exit 0 + fi + + if ! git config user.email "41898282+github-actions[bot]@users.noreply.github.com"; then + refresh_reason="git-config-failed" + exit 0 + fi + + if ! git remote set-url origin "https://x-access-token:${GH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git"; then + refresh_reason="remote-auth-failed" + exit 0 + fi + + if ! git fetch origin "pull/${PR_NUMBER}/head:refs/remotes/origin/external-pr-head-${PR_NUMBER}"; then + refresh_reason="fetch-external-failed" + exit 0 + fi + + external_ref="refs/remotes/origin/external-pr-head-${PR_NUMBER}" + branch_exists=false + if git ls-remote --exit-code --heads origin "${INTERNAL_BRANCH}" >/dev/null 2>&1; then + branch_exists=true + if ! git fetch origin "${INTERNAL_BRANCH}:refs/remotes/origin/${INTERNAL_BRANCH}"; then + refresh_reason="fetch-internal-failed" + exit 0 + fi + fi + + if [ "${branch_exists}" = false ]; then + if ! git checkout -B "${INTERNAL_BRANCH}" "${external_ref}"; then + refresh_reason="checkout-failed" + exit 0 + fi + + if ! git push --force-with-lease origin "HEAD:refs/heads/${INTERNAL_BRANCH}"; then + refresh_reason="push-failed" + exit 0 + fi + + refresh_status="updated" + refresh_reason="" + exit 0 + fi + + if ! git checkout -B "${INTERNAL_BRANCH}" "refs/remotes/origin/${INTERNAL_BRANCH}"; then + refresh_reason="checkout-failed" + exit 0 + fi + + if [ -z "${PREVIOUS_SOURCE_SHA}" ]; then + refresh_reason="missing-previous-source" + exit 0 + fi + + if git rebase --onto "${external_ref}" "${PREVIOUS_SOURCE_SHA}" "${INTERNAL_BRANCH}"; then + if ! git push --force-with-lease origin "HEAD:refs/heads/${INTERNAL_BRANCH}"; then + refresh_reason="push-failed" + exit 0 + fi + + refresh_status="updated" + refresh_reason="" + exit 0 + fi + + git rebase --abort || true + refresh_reason="rebase-conflict" + + - name: Finalize approved claim + id: finalize-claim + if: always() && steps.prepare-claim.outputs.should-claim == 'true' + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const lib = eval(process.env.ECPR_LIB); + const result = await lib.finalizeClaim({ + github, + context, + input: { + prNumber: Number('${{ steps.prepare-claim.outputs.pr-number }}'), + sourceSha: ${{ toJson(steps.prepare-claim.outputs.source-sha) }}, + branch: ${{ toJson(steps.prepare-claim.outputs.branch) }}, + claimer: ${{ toJson(steps.prepare-claim.outputs.claimer) }}, + title: ${{ toJson(steps.prepare-claim.outputs.title) }}, + body: ${{ toJson(steps.prepare-claim.outputs.body) }}, + existingNumber: ${{ toJson(steps.prepare-claim.outputs.owned-pr-number) }}, + existingMerged: '${{ steps.prepare-claim.outputs.owned-pr-merged }}' === 'true', + refreshStatus: ${{ toJson(steps.refresh-branch.outputs.refresh-status) }}, + refreshReason: ${{ toJson(steps.refresh-branch.outputs.reason) }}, + }, + }); + core.setOutput('claimed', result?.claimed ? 'true' : 'false'); + core.setOutput('owned-pr-number', result?.ownedPrNumber || ''); + + - name: Trigger claimed PR CI + if: steps.finalize-claim.outputs.claimed == 'true' + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + await github.rest.actions.createWorkflowDispatch({ + owner: context.repo.owner, + repo: context.repo.repo, + workflow_id: 'ci.yml', + ref: ${{ toJson(steps.prepare-claim.outputs.branch) }}, + inputs: { + pull_request_number: ${{ toJson(steps.finalize-claim.outputs.owned-pr-number) }}, + }, + }); + + sync-owned-pr: + if: github.event_name == 'pull_request_target' && github.event.pull_request.head.repo.full_name == github.repository && (github.event.action == 'closed' || github.event.action == 'reopened') + runs-on: ubuntu-latest + steps: + - name: Sync mirrored PR lifecycle + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const lib = eval(process.env.ECPR_LIB); + await lib.syncOwnedPr({ github, context }); diff --git a/.github/workflows/feature-parity.yml b/.github/workflows/feature-parity.yml new file mode 100644 index 000000000..bd388d584 --- /dev/null +++ b/.github/workflows/feature-parity.yml @@ -0,0 +1,147 @@ +name: Feature Parity + +on: + pull_request: + types: + - opened + - synchronize + - labeled + - unlabeled + paths-ignore: + - "packages/docs/**" + +jobs: + check-parity-label: + runs-on: ubuntu-latest + if: github.event.action == 'labeled' && github.event.label.name == 'parity' + permissions: + contents: read + pull-requests: write + issues: write + steps: + - name: Check out repository code + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + + - name: Check user permissions + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const { data: permission } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: context.repo.owner, + repo: context.repo.repo, + username: context.actor + }); + + const hasWriteAccess = ['admin', 'write'].includes(permission.permission); + + if (!hasWriteAccess) { + // Remove the parity label if user doesn't have write access + await github.rest.issues.removeLabel({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + name: 'parity' + }); + + // Add a comment explaining why the label was removed + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + body: `❌ **Parity Label Removed**\n\n@${context.actor}, you do not have sufficient permissions to add the 'parity' label. Only users with write access can trigger feature parity issues.\n\nIf you believe this feature should be implemented in the Python SDK, please ask a maintainer to add the label.` + }); + + throw new Error(`User ${context.actor} does not have write access to add parity label`); + } + + console.log(`User ${context.actor} has ${permission.permission} access - proceeding with parity workflow`); + + - name: Generate GitHub App token + id: generate-token + uses: actions/create-github-app-token@d72941d797fd3113feb6b93fd0dec494b13a2547 # v1.12.0 + with: + app-id: ${{ secrets.PARITY_APP_ID }} + private-key: ${{ secrets.PARITY_APP_PRIVATE_KEY }} + owner: browserbase + repositories: stagehand + + - name: Create issue in Python SDK repository + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 + with: + github-token: ${{ steps.generate-token.outputs.token }} + script: | + const { data: pullRequest } = await github.rest.pulls.get({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: context.issue.number, + }); + + // Get PR comments for additional context + const { data: comments } = await github.rest.issues.listComments({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + }); + + // Format comments for the issue description + let commentsSection = ''; + if (comments.length > 0) { + commentsSection = '\n\n## Recent Comments\n\n'; + comments.slice(-3).forEach(comment => { + commentsSection += `**@${comment.user.login}** commented:\n`; + commentsSection += `${comment.body.substring(0, 500)}${comment.body.length > 500 ? '...' : ''}\n\n`; + }); + } + + // Get list of changed files for context + const { data: files } = await github.rest.pulls.listFiles({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: context.issue.number, + }); + + const changedFiles = files.map(file => `- \`${file.filename}\``).join('\n'); + + const issueTitle = `[Feature Parity] ${pullRequest.title}`; + const issueBody = `## Feature Parity Request + + This issue was automatically created from a pull request in the TypeScript Stagehand repository that was labeled with 'parity'. + + ### Original PR Details + - **PR**: #${context.issue.number} - ${pullRequest.title} + - **Author**: @${pullRequest.user.login} + - **Link**: ${pullRequest.html_url} + + ### Description + ${pullRequest.body || 'No description provided.'} + + ### Changed Files + ${changedFiles} + + ${commentsSection} + + ### Action Required + Please review the changes in the original PR and implement equivalent functionality in the Python SDK if applicable. + + --- + *This issue was automatically generated by the Feature Parity workflow.*`; + + // Create the issue in the Python repository + const { data: issue } = await github.rest.issues.create({ + owner: 'browserbase', + repo: 'stagehand-python', + title: issueTitle, + body: issueBody, + labels: ['parity'] + }); + + console.log(`Created issue: ${issue.html_url}`); + + // Add a comment to the original PR confirming the issue was created + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + body: `🔄 **Feature Parity Issue Created**\n\nAn issue has been automatically created in the Python SDK repository to track parity implementation:\n${issue.html_url}` + }); diff --git a/.github/workflows/manual-evals.yml b/.github/workflows/manual-evals.yml new file mode 100644 index 000000000..540d32a9a --- /dev/null +++ b/.github/workflows/manual-evals.yml @@ -0,0 +1,281 @@ +name: Manual Evals + +on: + workflow_dispatch: + inputs: + eval_suite: + description: Eval suite to run. + required: true + type: choice + options: + - agent + - all + - external_agent_benchmarks + external_benchmark: + description: External agent benchmark to run when eval_suite is external_agent_benchmarks. + required: false + default: webvoyager + type: choice + options: + - webvoyager + - onlineMind2Web + - webtailbench + model: + description: Provider-qualified model name, for example openai/gpt-5.4-mini. + required: true + type: string + agent_mode: + description: Agent mode for agent evals. Auto lets the eval runner infer the mode from the model. + required: false + default: auto + type: choice + options: + - auto + - dom + - hybrid + - cua + trials: + description: Number of trials per eval. + required: false + default: 1 + type: number + concurrency: + description: Maximum eval concurrency. + required: false + default: 50 + type: number + +permissions: + contents: read + +env: + BROWSERBASE_FLOW_LOGS: "1" + LLM_MAX_MS: "15000" + BROWSERBASE_REGION_DISTRIBUTION: "us-west-2=30,us-east-1=30,eu-central-1=20,ap-southeast-1=20" + PUPPETEER_SKIP_DOWNLOAD: "1" + PLAYWRIGHT_SKIP_DOWNLOAD: "1" + TURBO_TELEMETRY_DISABLED: "1" + +jobs: + prepare: + name: Prepare Eval Matrix + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.matrix.outputs.matrix }} + steps: + - id: matrix + name: Build eval matrix + shell: bash + env: + EVAL_SUITE: ${{ inputs.eval_suite }} + EXTERNAL_BENCHMARK: ${{ inputs.external_benchmark }} + run: | + set -euo pipefail + + case "$EVAL_SUITE" in + agent) + matrix='[{"target":"agent","label":"agent","safe_target":"agent"}]' + ;; + all) + matrix='[ + {"target":"act","label":"act","safe_target":"act"}, + {"target":"extract","label":"extract","safe_target":"extract"}, + {"target":"observe","label":"observe","safe_target":"observe"} + ]' + ;; + external_agent_benchmarks) + case "$EXTERNAL_BENCHMARK" in + webvoyager|onlineMind2Web|webtailbench) + ;; + *) + echo "Unsupported external benchmark: $EXTERNAL_BENCHMARK" >&2 + exit 1 + ;; + esac + + matrix=$(jq -nc \ + --arg target "agent/${EXTERNAL_BENCHMARK}" \ + --arg label "external-agent-benchmark-${EXTERNAL_BENCHMARK}" \ + --arg safe_target "agent-${EXTERNAL_BENCHMARK}" \ + '[{target: $target, label: $label, safe_target: $safe_target}]') + ;; + *) + echo "Unsupported eval suite: $EVAL_SUITE" >&2 + exit 1 + ;; + esac + + matrix=$(jq -c . <<< "$matrix") + echo "matrix=$matrix" >> "$GITHUB_OUTPUT" + + run-build: + name: Build + runs-on: ubuntu-latest + steps: + - name: Check out repository code + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + + - uses: ./.github/actions/setup-node-pnpm-turbo + with: + use-prebuilt-artifacts: "false" + node-version: 20.x + + - name: Run Build + run: pnpm exec turbo run build + + - name: Save Turbo cache + if: always() + uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: .turbo + key: ${{ runner.os }}-turbo-${{ hashFiles('pnpm-lock.yaml', 'pnpm-workspace.yaml', 'package.json', 'turbo.json') }}-${{ github.sha }} + + - name: Upload build artifacts + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: build-artifacts + include-hidden-files: true + path: | + package.json + packages/core/dist/** + packages/core/lib/version.ts + packages/core/lib/dom/build/** + packages/core/lib/v3/dom/build/** + packages/cli/dist/** + packages/evals/dist/** + retention-days: 1 + + run-evals: + name: evals/${{ matrix.label }} + runs-on: ubuntu-latest + needs: + - prepare + - run-build + timeout-minutes: 90 + strategy: + fail-fast: false + matrix: + include: ${{ fromJson(needs.prepare.outputs.matrix) }} + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + GOOGLE_GENERATIVE_AI_API_KEY: ${{ secrets.GOOGLE_GENERATIVE_AI_API_KEY }} + BRAINTRUST_API_KEY: ${{ secrets.BRAINTRUST_API_KEY }} + BROWSERBASE_API_KEY: ${{ secrets.BROWSERBASE_API_KEY }} + BROWSERBASE_PROJECT_ID: ${{ secrets.BROWSERBASE_PROJECT_ID }} + STAGEHAND_BROWSER_TARGET: browserbase + STAGEHAND_SERVER_TARGET: local + steps: + - name: Check out repository code + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + + - uses: ./.github/actions/setup-node-pnpm-turbo + with: + use-prebuilt-artifacts: "true" + restore-turbo-cache: "false" + node-version: 20.x + + - name: Select Browserbase region + uses: ./.github/actions/select-browserbase-region + with: + distribution: ${{ env.BROWSERBASE_REGION_DISTRIBUTION }} + + - name: Run Evals - ${{ matrix.label }} + id: run-evals + env: + NODE_V8_COVERAGE: coverage/manual-evals/${{ matrix.safe_target }} + EVAL_TARGET: ${{ matrix.target }} + EVAL_MODEL: ${{ inputs.model }} + EVAL_TRIALS: ${{ inputs.trials }} + EVAL_CONCURRENCY: ${{ inputs.concurrency }} + EVAL_AGENT_MODE: ${{ inputs.agent_mode }} + shell: bash + run: | + set +e + + # The GitHub iOS app serializes `type: number` workflow_dispatch + # inputs as doubles ("1.0"/"50.0") regardless of whether the + # workflow needs an integer. The evals CLI's `parsePositiveInteger` + # (regex /^[0-9]+$/) rejects the decimal form, so strip a single + # trailing `.0` here before passing the values along. + EVAL_TRIALS="${EVAL_TRIALS%.0}" + EVAL_CONCURRENCY="${EVAL_CONCURRENCY%.0}" + + agent_mode_args=() + if [[ -n "${EVAL_AGENT_MODE:-}" && "$EVAL_AGENT_MODE" != "auto" ]]; then + agent_mode_args=(--agent-mode "$EVAL_AGENT_MODE") + fi + + pnpm exec turbo run test:evals --only --filter=@browserbasehq/stagehand-evals -- \ + run "$EVAL_TARGET" \ + -e browserbase \ + -t "$EVAL_TRIALS" \ + -c "$EVAL_CONCURRENCY" \ + -m "$EVAL_MODEL" \ + "${agent_mode_args[@]}" + eval_status=$? + set -e + + if [ -f eval-summary.json ]; then + { + echo "summary_text<> "$GITHUB_OUTPUT" + fi + + exit "$eval_status" + + - name: Log Evals Performance - ${{ matrix.label }} + if: always() + env: + EVAL_STDOUT_SUMMARY: ${{ steps.run-evals.outputs.summary_text }} + shell: bash + run: | + if [ -n "${EVAL_STDOUT_SUMMARY:-}" ]; then + echo "### Evals Summary (${{ matrix.label }})" >> "$GITHUB_STEP_SUMMARY" + echo '```' >> "$GITHUB_STEP_SUMMARY" + printf '%s\n' "$EVAL_STDOUT_SUMMARY" >> "$GITHUB_STEP_SUMMARY" + echo '```' >> "$GITHUB_STEP_SUMMARY" + fi + + if [ ! -f eval-summary.json ]; then + echo "Eval summary not found for ${{ matrix.label }}. Failing workflow." + exit 1 + fi + + experimentUrl=$(jq -r '.experimentUrl // empty' eval-summary.json) + if [ -n "$experimentUrl" ]; then + echo "View results at ${experimentUrl}" + else + experimentName=$(jq -r '.experimentName' eval-summary.json) + echo "View results at https://www.braintrust.dev/app/Browserbase/p/stagehand/experiments/${experimentName}" + fi + + primary_score=$(jq -r '(.scores["Exact match"].score // .scores["Pass"].score // empty)' eval-summary.json) + if [ -n "$primary_score" ]; then + primary_score_percent=$(jq -rn --arg score "$primary_score" '$score | tonumber * 100') + printf '${{ matrix.label }} Braintrust score: %.2f%%\n' "$primary_score_percent" + else + echo "Braintrust primary score not found in eval summary." + fi + + - uses: ./.github/actions/upload-ctrf-report + if: always() + with: + name: ctrf/manual-evals/${{ matrix.safe_target }}.json + path: ctrf/evals/${{ matrix.safe_target }}.json + + - uses: ./.github/actions/upload-v8-coverage + if: always() + with: + name: coverage/manual-evals/${{ matrix.safe_target }} + path: coverage/manual-evals/${{ matrix.safe_target }} diff --git a/.github/workflows/publish-evals.yml b/.github/workflows/publish-evals.yml new file mode 100644 index 000000000..3d5e0b278 --- /dev/null +++ b/.github/workflows/publish-evals.yml @@ -0,0 +1,130 @@ +name: Publish Evals + +on: + workflow_dispatch: + inputs: + experiment: + description: Braintrust experiment name or UUID. + required: true + type: string + project: + description: Braintrust project containing the experiment. + required: false + default: stagehand + type: string + kv_key: + description: Upstash Redis key the UI reads. + required: false + default: stagehand:evals:latest + type: string + experiment_key_prefix: + description: Prefix for the secondary experiment-id key. + required: false + default: stagehand:evals:experiments + type: string + write_experiment_key: + description: Also write :. + required: false + default: true + type: boolean + dry_run: + description: Fetch and render payload without writing to Upstash. + required: false + default: false + type: boolean + +permissions: + contents: read + +env: + PUPPETEER_SKIP_DOWNLOAD: "1" + PLAYWRIGHT_SKIP_DOWNLOAD: "1" + TURBO_TELEMETRY_DISABLED: "1" + +jobs: + publish: + name: Publish Evals UI Data + runs-on: ubuntu-latest + steps: + - name: Check out repository code + uses: actions/checkout@v4 + + - uses: ./.github/actions/setup-node-pnpm-turbo + with: + use-prebuilt-artifacts: "false" + restore-turbo-cache: "false" + node-version: 20.x + + - name: Publish Braintrust experiment to Upstash + id: publish + env: + BRAINTRUST_API_KEY: ${{ secrets.BRAINTRUST_API_KEY }} + UPSTASH_REDIS_REST_URL: ${{ secrets.UPSTASH_REDIS_REST_URL }} + UPSTASH_REDIS_REST_TOKEN: ${{ secrets.UPSTASH_REDIS_REST_TOKEN }} + KV_REST_API_URL: ${{ secrets.KV_REST_API_URL }} + KV_REST_API_TOKEN: ${{ secrets.KV_REST_API_TOKEN }} + EXPERIMENT: ${{ inputs.experiment }} + PROJECT: ${{ inputs.project }} + KV_KEY: ${{ inputs.kv_key }} + EXPERIMENT_KEY_PREFIX: ${{ inputs.experiment_key_prefix }} + WRITE_EXPERIMENT_KEY: ${{ inputs.write_experiment_key }} + DRY_RUN: ${{ inputs.dry_run }} + shell: bash + run: | + set -euo pipefail + + args=( + --experiment "$EXPERIMENT" + --project "$PROJECT" + --key "$KV_KEY" + --experiment-key-prefix "$EXPERIMENT_KEY_PREFIX" + --out evals-ui-data.json + ) + + if [[ "$WRITE_EXPERIMENT_KEY" != "true" ]]; then + args+=(--no-experiment-key) + fi + + if [[ "$DRY_RUN" == "true" ]]; then + args+=(--dry-run) + fi + + pnpm --filter @browserbasehq/stagehand-evals exec tsx \ + scripts/publish-braintrust-ui-data.ts \ + "${args[@]}" | tee publish-evals-ui-data-output.json + + { + echo "summary<> "$GITHUB_OUTPUT" + + - name: Add publish summary + if: always() + env: + PUBLISH_SUMMARY: ${{ steps.publish.outputs.summary }} + shell: bash + run: | + if [ -n "${PUBLISH_SUMMARY:-}" ]; then + echo "### Published Evals UI Data" >> "$GITHUB_STEP_SUMMARY" + echo '```' >> "$GITHUB_STEP_SUMMARY" + printf '%s\n' "$PUBLISH_SUMMARY" >> "$GITHUB_STEP_SUMMARY" + echo '```' >> "$GITHUB_STEP_SUMMARY" + fi + + - name: Upload generated payload + if: always() + uses: actions/upload-artifact@v4 + with: + name: evals-ui-data + path: | + evals-ui-data.json + publish-evals-ui-data-output.json diff --git a/.github/workflows/release-cli.yml b/.github/workflows/release-cli.yml new file mode 100644 index 000000000..5d09bf70a --- /dev/null +++ b/.github/workflows/release-cli.yml @@ -0,0 +1,161 @@ +name: Prepare CLI Release + +on: + workflow_dispatch: + +permissions: + contents: write + pull-requests: write + +concurrency: ${{ github.workflow }}-${{ github.ref }} + +jobs: + release-cli: + name: Prepare browse release PR + if: github.ref == 'refs/heads/main' + runs-on: ubuntu-latest + steps: + - name: Checkout Repo + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + fetch-depth: 0 + + - uses: ./.github/actions/setup-node-pnpm-turbo + with: + use-prebuilt-artifacts: "false" + + - name: Check for browse changesets + id: check + run: | + # Only look at changeset markdown files (skip config, README, etc.) + CLI_CHANGESETS="" + for f in .changeset/*.md; do + [ "$f" = ".changeset/README.md" ] && continue + if grep -qE '"browse"' "$f" 2>/dev/null; then + CLI_CHANGESETS="${CLI_CHANGESETS} ${f}" + fi + done + + if [ -z "$CLI_CHANGESETS" ]; then + echo "No pending browse changesets found." + echo "has_changesets=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + + echo "Found browse changesets:${CLI_CHANGESETS}" + echo "has_changesets=true" >> "$GITHUB_OUTPUT" + + # Fail if any changeset references browse AND another package. + # Mixed changesets must be split so both release flows get their bumps. + for f in $CLI_CHANGESETS; do + if grep -qE '"@browserbasehq/(stagehand|stagehand-server-v3|stagehand-server-v4|stagehand-evals|stagehand-docs)"' "$f"; then + echo "::error::Mixed changeset found: $f references browse AND other packages. Split it into separate changesets so both release workflows can consume their respective bumps." + exit 1 + fi + done + + - name: Version browse + if: steps.check.outputs.has_changesets == 'true' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + # Back up config and package.json + cp .changeset/config.json .changeset/config.json.bak + cp packages/cli/package.json packages/cli/package.json.bak + + # Temporarily strip the workspace dep on stagehand so changesets + # doesn't see a dependency edge (it validates that non-ignored + # packages can't depend on ignored ones). The dep is restored + # afterwards; pnpm pack rewrites it to the published version at + # publish time. + node -e " + const fs = require('fs'); + const pkg = JSON.parse(fs.readFileSync('packages/cli/package.json', 'utf8')); + delete pkg.dependencies['@browserbasehq/stagehand']; + if (pkg.devDependencies) delete pkg.devDependencies['@browserbasehq/stagehand']; + fs.writeFileSync('packages/cli/package.json', JSON.stringify(pkg, null, 2) + '\n'); + + // Ignore every workspace package except browse so 'changeset version' + // only bumps browse. Keep this list in sync with the packages that + // actually exist on disk -- changeset validation fails if an entry + // here doesn't resolve to a real package. + const config = JSON.parse(fs.readFileSync('.changeset/config.json', 'utf8')); + config.ignore = [ + '@browserbasehq/stagehand', + '@browserbasehq/stagehand-server-v3', + '@browserbasehq/stagehand-evals', + '@browserbasehq/stagehand-docs' + ]; + fs.writeFileSync('.changeset/config.json', JSON.stringify(config, null, 2) + '\n'); + " + + # Version only browse (consumes CLI-only changesets) + pnpm changeset version + + # Restore config and merge the version bump into the original package.json + node -e " + const fs = require('fs'); + const bumped = JSON.parse(fs.readFileSync('packages/cli/package.json', 'utf8')); + const backup = JSON.parse(fs.readFileSync('packages/cli/package.json.bak', 'utf8')); + backup.version = bumped.version; + fs.writeFileSync('packages/cli/package.json', JSON.stringify(backup, null, 2) + '\n'); + " + mv .changeset/config.json.bak .changeset/config.json + rm packages/cli/package.json.bak + + - name: Create or update release pull request + if: steps.check.outputs.has_changesets == 'true' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + + VERSION=$(node -p "require('./packages/cli/package.json').version") + BRANCH="release/browse" + PR_TITLE="Release browse@${VERSION}" + PR_BODY_FILE=$(mktemp) + cat > "$PR_BODY_FILE" <= 11.5.1; npm 12+ requires + # node >= 22 and this job runs node 20, so pin the last known-good + # 11.x exactly. Unpin (back to npm@latest) when the job moves to + # node >= 22. + run: npm install -g npm@11.18.0 + + - name: Run Lint & Build + run: pnpm exec turbo run lint && pnpm exec turbo run build + + - name: Check for actionable changesets + id: check_changesets run: | - rm -rf node_modules - rm -f package-lock.json - npm install + # Read the ignore list from .changeset/config.json + IGNORED=$(node -e " + const config = require('./.changeset/config.json'); + (config.ignore || []).forEach(p => console.log(p)); + ") - - name: Build - run: npm run build + HAS_ACTIONABLE=false + HAS_ANY=false + for f in .changeset/*.md; do + [ "$f" = ".changeset/README.md" ] && continue + [ ! -f "$f" ] && continue + HAS_ANY=true + + # Extract package names from the changeset frontmatter + PACKAGES=$(sed -n '/^---$/,/^---$/{ /^---$/d; s/^"\(.*\)":.*/\1/p }' "$f") + + for pkg in $PACKAGES; do + if ! echo "$IGNORED" | grep -qxF "$pkg"; then + HAS_ACTIONABLE=true + break 2 + fi + done + done + + # Skip only when ALL pending changesets target ignored packages. + # When there are no changesets at all, the action must still run + # so it can publish freshly-versioned packages. + if [ "$HAS_ANY" = "true" ] && [ "$HAS_ACTIONABLE" = "false" ]; then + echo "All pending changesets target ignored packages — skipping changesets action." + echo "should_run=false" >> "$GITHUB_OUTPUT" + else + echo "should_run=true" >> "$GITHUB_OUTPUT" + fi - name: Create Release Pull Request or Publish to npm id: changesets - uses: changesets/action@v1 + uses: changesets/action@63a615b9cd06ba9a3e6d13796c7fbcb080a60a0b # v1.8.0 + if: steps.check_changesets.outputs.should_run == 'true' with: - publish: npm run release + publish: pnpm run release + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Publish already-versioned packages + if: steps.check_changesets.outputs.should_run != 'true' + run: pnpm run release env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + + - name: Check if browse publish is needed + if: github.ref == 'refs/heads/main' + id: browse_cli + run: | + # Restore working tree — changesets/action may have switched branches + git checkout ${{ github.sha }} + + if git diff --quiet ${{ github.sha }}^ ${{ github.sha }} -- packages/cli/package.json; then + echo "browse version did not change on this push." + echo "should_publish=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + + OLD_VERSION=$(git show ${{ github.sha }}^:packages/cli/package.json | node -pe "JSON.parse(require('fs').readFileSync(0, 'utf8')).version") + NEW_VERSION=$(git show ${{ github.sha }}:packages/cli/package.json | node -pe "JSON.parse(require('fs').readFileSync(0, 'utf8')).version") + echo "version=${NEW_VERSION}" >> "$GITHUB_OUTPUT" + + if [ "$OLD_VERSION" = "$NEW_VERSION" ]; then + echo "browse version did not change on this push." + echo "should_publish=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + + echo "should_publish=true" >> "$GITHUB_OUTPUT" + + if npm view "browse@${NEW_VERSION}" version >/dev/null 2>&1; then + echo "browse ${NEW_VERSION} is already published." + echo "already_published=true" >> "$GITHUB_OUTPUT" + else + echo "already_published=false" >> "$GITHUB_OUTPUT" + fi + + - name: Publish browse to npm + if: steps.browse_cli.outputs.should_publish == 'true' && steps.browse_cli.outputs.already_published != 'true' + run: | + cd packages/cli + PACK_DIR=$(mktemp -d) + trap 'rm -rf "$PACK_DIR"' EXIT + + TARBALL=$(pnpm pack --json --pack-destination "$PACK_DIR" | node -pe "JSON.parse(require('fs').readFileSync(0, 'utf8')).filename") + npm publish "$TARBALL" --provenance --access public --tag latest + + - name: Tag browse release + if: github.ref == 'refs/heads/main' && steps.browse_cli.outputs.should_publish == 'true' && steps.browse_cli.outputs.already_published != 'true' + run: | + TAG="browse@${{ steps.browse_cli.outputs.version }}" + if git rev-parse -q --verify "refs/tags/${TAG}" >/dev/null; then + echo "Tag ${TAG} already exists." + exit 0 + fi + + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git tag "$TAG" + git push origin "$TAG" + + - name: Set up QEMU + if: steps.browse_cli.outputs.should_publish == 'true' && steps.browse_cli.outputs.already_published != 'true' + uses: docker/setup-qemu-action@06116385d9baf250c9f4dcb4858b16962ea869c3 # v4.1.0 + + - name: Set up Docker Buildx + if: steps.browse_cli.outputs.should_publish == 'true' && steps.browse_cli.outputs.already_published != 'true' + uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0 + + - name: Log in to GitHub Container Registry + if: steps.browse_cli.outputs.should_publish == 'true' && steps.browse_cli.outputs.already_published != 'true' + uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push browse image to GHCR + if: steps.browse_cli.outputs.should_publish == 'true' && steps.browse_cli.outputs.already_published != 'true' + uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # v7.2.0 + with: + context: packages/cli + file: packages/cli/Dockerfile + platforms: linux/amd64,linux/arm64 + push: true + provenance: false + build-args: | + BROWSE_VERSION=${{ steps.browse_cli.outputs.version }} + tags: | + ghcr.io/browserbase/browse:${{ steps.browse_cli.outputs.version }} + ghcr.io/browserbase/browse:latest + + - name: Publish browse canary + if: github.ref == 'refs/heads/main' && steps.browse_cli.outputs.should_publish != 'true' + run: | + # Skip if no browse files changed in this push + if git diff --quiet ${{ github.sha }}^ ${{ github.sha }} -- packages/cli/; then + echo "No browse changes in this push, skipping canary." + exit 0 + fi + + cd packages/cli + BASE_VERSION=$(node -p "require('./package.json').version") + SHORT_SHA=$(echo "${{ github.sha }}" | cut -c1-7) + CANARY_VERSION="${BASE_VERSION}-alpha-${SHORT_SHA}" + + if npm view "browse@${CANARY_VERSION}" version >/dev/null 2>&1; then + echo "browse canary ${CANARY_VERSION} is already published." + exit 0 + fi + + # Temporarily set canary version for pnpm pack + node -e " + const pkg = require('./package.json'); + pkg.version = '${CANARY_VERSION}'; + require('fs').writeFileSync('package.json', JSON.stringify(pkg, null, 2) + '\n'); + " + + PACK_DIR=$(mktemp -d) + trap 'git checkout -- package.json; rm -rf "$PACK_DIR"' EXIT + + TARBALL=$(pnpm pack --json --pack-destination "$PACK_DIR" | node -pe "JSON.parse(require('fs').readFileSync(0, 'utf8')).filename") + npm publish "$TARBALL" --provenance --access public --tag alpha + + echo "Published browse@${CANARY_VERSION}" - name: Publish Canary if: github.ref == 'refs/heads/main' run: | - npm config set //registry.npmjs.org/:_authToken=${NODE_AUTH_TOKEN} git checkout main - npm run release-canary + pnpm run release-canary env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} diff --git a/.github/workflows/stagehand-server-v3-release.yml b/.github/workflows/stagehand-server-v3-release.yml new file mode 100644 index 000000000..b09c7b3b7 --- /dev/null +++ b/.github/workflows/stagehand-server-v3-release.yml @@ -0,0 +1,326 @@ +name: Release stagehand/server-v3 + +on: + push: + branches: + - main + paths: + - .changeset/** + - .github/workflows/stagehand-server-v3-release.yml + workflow_dispatch: + +permissions: + contents: write + +concurrency: ${{ github.workflow }}-${{ github.ref }} + +env: + OAS_PATH: packages/server-v3/openapi.v3.yaml + +jobs: + detect: + name: Detect server-v3 release (changesets) + runs-on: ubuntu-latest + outputs: + release: ${{ steps.meta.outputs.release }} + version: ${{ steps.meta.outputs.version }} + tag: ${{ steps.meta.outputs.tag }} + steps: + - name: Checkout repository + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + fetch-depth: 1 + fetch-tags: true + + - uses: ./.github/actions/setup-node-pnpm-turbo + env: + PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: "1" + with: + use-prebuilt-artifacts: "false" + + - name: Determine release metadata + id: meta + shell: bash + run: | + set -euo pipefail + + remote_tags="$(mktemp)" + git ls-remote --tags --refs origin 'refs/tags/stagehand-server-v3/v*' \ + | awk '{sub("refs/tags/", "", $2); print $2}' \ + > "${remote_tags}" + + latest_tag_file="$(mktemp)" + REMOTE_TAGS_FILE="${remote_tags}" LATEST_TAG_FILE="${latest_tag_file}" node <<'NODE' + const fs = require('fs'); + + const tags = fs + .readFileSync(process.env.REMOTE_TAGS_FILE, 'utf8') + .split('\n') + .map((line) => line.trim()) + .filter(Boolean); + + const parseTag = (tag) => { + const match = /^stagehand-server-v3\/v(\d+)\.(\d+)\.(\d+)$/.exec(tag); + if (!match) { + return null; + } + return match.slice(1).map((part) => Number(part)); + }; + + const compareVersions = (left, right) => { + for (let i = 0; i < 3; i += 1) { + if (left[i] !== right[i]) { + return left[i] - right[i]; + } + } + return 0; + }; + + let best = null; + for (const tag of tags) { + const version = parseTag(tag); + if (!version) { + continue; + } + if (best === null || compareVersions(version, best.version) > 0) { + best = { tag, version }; + } + } + + fs.writeFileSync(process.env.LATEST_TAG_FILE, best?.tag ?? '', 'utf8'); + NODE + latest_tag="$(cat "${latest_tag_file}")" + + changed_files="$(mktemp)" + if [ -n "${latest_tag}" ]; then + git fetch --force origin "refs/tags/${latest_tag}:refs/tags/${latest_tag}" + git diff --diff-filter=d --name-only "${latest_tag}"..HEAD -- '.changeset/*.md' > "${changed_files}" + else + find .changeset -maxdepth 1 -name '*.md' -print | sort > "${changed_files}" + fi + + LATEST_TAG="${latest_tag}" REMOTE_TAGS_FILE="${remote_tags}" CHANGED_CHANGESET_FILES="${changed_files}" node <<'NODE' + const fs = require('fs'); + const path = require('path'); + + const packageName = '@browserbasehq/stagehand-server-v3'; + const latestTag = process.env.LATEST_TAG || ''; + const remoteTags = fs + .readFileSync(process.env.REMOTE_TAGS_FILE, 'utf8') + .split('\n') + .map((line) => line.trim()) + .filter(Boolean); + const changesetFiles = fs + .readFileSync(process.env.CHANGED_CHANGESET_FILES, 'utf8') + .split('\n') + .map((line) => line.trim()) + .filter(Boolean) + .filter((file) => path.basename(file) !== 'config.json'); + + const releasePriority = { patch: 0, minor: 1, major: 2 }; + const compareVersions = (left, right) => { + for (let i = 0; i < 3; i += 1) { + if (left[i] !== right[i]) { + return left[i] - right[i]; + } + } + return 0; + }; + const parseTag = (tag) => { + const match = /^stagehand-server-v3\/v(\d+)\.(\d+)\.(\d+)$/.exec(tag); + if (!match) { + return null; + } + return match.slice(1).map((part) => Number(part)); + }; + + let highestRemoteVersion = null; + for (const tag of remoteTags) { + const version = parseTag(tag); + if (!version) { + continue; + } + if (highestRemoteVersion === null || compareVersions(version, highestRemoteVersion) > 0) { + highestRemoteVersion = version; + } + } + + let highestReleaseType = null; + + for (const file of changesetFiles) { + const content = fs.readFileSync(file, 'utf8'); + const match = /^---\n([\s\S]*?)\n---/m.exec(content); + if (!match) { + continue; + } + + for (const rawLine of match[1].split('\n')) { + const line = rawLine.trim(); + if (!line || line.startsWith('#')) { + continue; + } + + const releaseMatch = /^"([^"]+)"\s*:\s*(patch|minor|major)\s*$/.exec(line); + if (!releaseMatch) { + continue; + } + + const [, releasePackage, releaseType] = releaseMatch; + if (releasePackage !== packageName) { + continue; + } + + if ( + highestReleaseType === null || + releasePriority[releaseType] > releasePriority[highestReleaseType] + ) { + highestReleaseType = releaseType; + } + } + } + + const shouldRelease = highestReleaseType !== null; + let version = ''; + + if (shouldRelease) { + if (!latestTag || highestRemoteVersion === null) { + throw new Error( + 'No existing remote stagehand-server-v3 release tag was found. Refusing to bootstrap from 0.0.0.', + ); + } + + const parts = [...highestRemoteVersion]; + if (highestReleaseType === 'major') { + parts[0] += 1; + parts[1] = 0; + parts[2] = 0; + } else if (highestReleaseType === 'minor') { + parts[1] += 1; + parts[2] = 0; + } else { + parts[2] += 1; + } + + if (compareVersions(parts, highestRemoteVersion) <= 0) { + throw new Error( + `Computed stagehand-server-v3 version ${parts.join('.')} does not advance remote latest ${highestRemoteVersion.join('.')}.`, + ); + } + + version = parts.join('.'); + } + + const release = shouldRelease ? 'true' : 'false'; + const tag = `stagehand-server-v3/v${version}`; + + const out = process.env.GITHUB_OUTPUT; + fs.appendFileSync(out, `release=${release}\n`); + fs.appendFileSync(out, `version=${version}\n`); + fs.appendFileSync(out, `tag=${tag}\n`); + NODE + + - name: Create stagehand/server-v3 tag + if: steps.meta.outputs.release == 'true' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + shell: bash + run: | + set -euo pipefail + + TAG="${{ steps.meta.outputs.tag }}" + VERSION="${{ steps.meta.outputs.version }}" + TARGET_SHA="${{ github.sha }}" + + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + + # Try to fetch the tag if it exists on remote; ignore failure for new tags + git fetch --force origin "refs/tags/${TAG}:refs/tags/${TAG}" 2>/dev/null || true + if git rev-parse -q --verify "refs/tags/${TAG}" >/dev/null; then + echo "Tag already exists: ${TAG}" + exit 0 + fi + + git tag -a "${TAG}" "${TARGET_SHA}" -m "stagehand/server-v3 v${VERSION}" + git push origin "${TAG}" + + build_binaries: + name: Build SEA binaries + needs: detect + if: needs.detect.outputs.release == 'true' + uses: ./.github/workflows/stagehand-server-v3-sea-build.yml + with: + matrix: | + [ + {"os":"ubuntu-latest","platform":"linux","arch":"x64","binary_name":"stagehand-server-v3-linux-x64","include_sourcemaps":false}, + {"os":"ubuntu-24.04-arm","platform":"linux","arch":"arm64","binary_name":"stagehand-server-v3-linux-arm64","include_sourcemaps":false}, + {"os":"macos-15","platform":"darwin","arch":"arm64","binary_name":"stagehand-server-v3-darwin-arm64","include_sourcemaps":false}, + {"os":"macos-15-intel","platform":"darwin","arch":"x64","binary_name":"stagehand-server-v3-darwin-x64","include_sourcemaps":false}, + {"os":"windows-latest","platform":"win32","arch":"x64","binary_name":"stagehand-server-v3-win32-x64.exe","include_sourcemaps":false}, + {"os":"windows-11-arm","platform":"win32","arch":"arm64","binary_name":"stagehand-server-v3-win32-arm64.exe","include_sourcemaps":false} + ] + + release: + name: Publish GitHub Release + needs: [detect, build_binaries] + if: needs.detect.outputs.release == 'true' + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + fetch-depth: 1 + fetch-tags: false + + - name: Prepare release assets directory + run: mkdir -p release-assets + + - name: Prepare stagehand/server-v3 release assets + run: | + set -euo pipefail + cp "${{ env.OAS_PATH }}" "release-assets/openapi.v3.stagehand-server-v3-${{ needs.detect.outputs.version }}.yaml" + + - name: Download SEA binary artifacts + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + pattern: stagehand-server-v3-* + path: . + merge-multiple: true + + - name: Collect SEA binaries + shell: bash + run: | + set -euo pipefail + shopt -s nullglob + for f in packages/server-v3/dist/sea/stagehand-server-v3-*; do + cp "$f" release-assets/ + done + + - name: Create checksums + shell: bash + run: | + set -euo pipefail + cd release-assets + # Only checksum binaries (exclude openapi yaml). Avoid failing if no matches. + shopt -s nullglob + files=(stagehand-server-v3-*) + bins=() + for f in "${files[@]}"; do + [[ "$f" == *openapi* ]] && continue + [[ -f "$f" ]] && bins+=("$f") + done + : > checksums.sha256 + if [ "${#bins[@]}" -gt 0 ]; then + shasum -a 256 "${bins[@]}" > checksums.sha256 + fi + + - name: Publish stagehand/server-v3 GitHub release + uses: softprops/action-gh-release@3bb12739c298aeb8a4eeaf626c5b8d85266b0e65 # v2.6.2 + with: + tag_name: ${{ needs.detect.outputs.tag }} + name: stagehand/server-v3 v${{ needs.detect.outputs.version }} + generate_release_notes: true + files: | + release-assets/openapi.v3.stagehand-server-v3-${{ needs.detect.outputs.version }}.yaml + release-assets/stagehand-server-v3-* + release-assets/checksums.sha256 diff --git a/.github/workflows/stagehand-server-v3-sea-build.yml b/.github/workflows/stagehand-server-v3-sea-build.yml new file mode 100644 index 000000000..4863318df --- /dev/null +++ b/.github/workflows/stagehand-server-v3-sea-build.yml @@ -0,0 +1,180 @@ +name: Stagehand Server v3 SEA Build + +on: + workflow_call: + inputs: + matrix: + description: "JSON matrix include list for SEA binaries." + required: false + type: string + default: | + [ + {"os":"ubuntu-latest","platform":"linux","arch":"x64","binary_name":"stagehand-server-v3-linux-x64","include_sourcemaps":false}, + {"os":"ubuntu-24.04-arm","platform":"linux","arch":"arm64","binary_name":"stagehand-server-v3-linux-arm64","include_sourcemaps":false}, + {"os":"macos-15","platform":"darwin","arch":"arm64","binary_name":"stagehand-server-v3-darwin-arm64","include_sourcemaps":false}, + {"os":"macos-15-intel","platform":"darwin","arch":"x64","binary_name":"stagehand-server-v3-darwin-x64","include_sourcemaps":false}, + {"os":"windows-latest","platform":"win32","arch":"x64","binary_name":"stagehand-server-v3-win32-x64.exe","include_sourcemaps":false}, + {"os":"windows-11-arm","platform":"win32","arch":"arm64","binary_name":"stagehand-server-v3-win32-arm64.exe","include_sourcemaps":false} + ] + use-prebuilt-artifacts: + description: "Whether to download pre-built package artifacts." + required: false + type: string + default: "false" + restore-turbo-cache: + description: "Whether to restore local .turbo cache." + required: false + type: string + default: "true" + node-version: + description: "Node.js version for setup." + required: false + type: string + default: "20.x" + upload-only-binary: + description: "Upload only this binary (empty => upload all)." + required: false + type: string + default: "" + workflow_dispatch: + inputs: + matrix: + description: "JSON matrix include list for SEA binaries." + required: false + default: | + [ + {"os":"ubuntu-latest","platform":"linux","arch":"x64","binary_name":"stagehand-server-v3-linux-x64","include_sourcemaps":false}, + {"os":"ubuntu-24.04-arm","platform":"linux","arch":"arm64","binary_name":"stagehand-server-v3-linux-arm64","include_sourcemaps":false}, + {"os":"macos-15","platform":"darwin","arch":"arm64","binary_name":"stagehand-server-v3-darwin-arm64","include_sourcemaps":false}, + {"os":"macos-15-intel","platform":"darwin","arch":"x64","binary_name":"stagehand-server-v3-darwin-x64","include_sourcemaps":false}, + {"os":"windows-latest","platform":"win32","arch":"x64","binary_name":"stagehand-server-v3-win32-x64.exe","include_sourcemaps":false}, + {"os":"windows-11-arm","platform":"win32","arch":"arm64","binary_name":"stagehand-server-v3-win32-arm64.exe","include_sourcemaps":false} + ] + use-prebuilt-artifacts: + description: "Whether to download pre-built package artifacts." + required: false + type: string + default: "false" + restore-turbo-cache: + description: "Whether to restore local .turbo cache." + required: false + type: string + default: "true" + node-version: + description: "Node.js version for setup." + required: false + type: string + default: "20.x" + upload-only-binary: + description: "Upload only this binary (empty => upload all)." + required: false + type: string + default: "" + +jobs: + build_binaries: + name: Build SEA binaries (${{ matrix.binary_name }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJson(inputs.matrix) }} + + steps: + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + fetch-depth: 1 + fetch-tags: false + + - uses: ./.github/actions/setup-node-pnpm-turbo + env: + PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: "1" + PLAYWRIGHT_SKIP_DOWNLOAD: "1" + PUPPETEER_SKIP_DOWNLOAD: "1" + with: + use-prebuilt-artifacts: ${{ inputs.use-prebuilt-artifacts }} + restore-turbo-cache: ${{ inputs.restore-turbo-cache }} + node-version: ${{ inputs.node-version }} + + - name: Build SEA binary (ESM) + env: + SEA_TARGET_PLATFORM: ${{ matrix.platform }} + SEA_TARGET_ARCH: ${{ matrix.arch }} + SEA_BINARY_NAME: ${{ matrix.binary_name }} + SEA_INCLUDE_SOURCEMAPS: ${{ matrix.include_sourcemaps && '1' || '0' }} + run: pnpm exec turbo run build:sea:esm --filter=@browserbasehq/stagehand-server-v3 + + - name: Verify SEA binary exists + shell: bash + run: | + test -f "packages/server-v3/dist/sea/${{ matrix.binary_name }}" + + - name: Verify SEA binary launches cleanly + shell: bash + env: + RUNNER_ARCH: ${{ runner.arch }} + run: | + set -euo pipefail + + binary="packages/server-v3/dist/sea/${{ matrix.binary_name }}" + matrix_arch="${{ matrix.arch }}" + runner_arch="$(echo "${RUNNER_ARCH}" | tr '[:upper:]' '[:lower:]')" + + if [[ "${matrix_arch}" != "${runner_arch}" ]]; then + echo "Runner arch (${runner_arch}) does not match matrix arch (${matrix_arch})." + echo "Launch verification must run on same-arch runners." + exit 1 + fi + + if [[ "${{ matrix.platform }}" != "win32" ]]; then + chmod +x "${binary}" + fi + + port="$((30000 + RANDOM % 10000))" + log_file="$(mktemp)" + launched="false" + + cleanup() { + if [[ -n "${pid:-}" ]] && kill -0 "${pid}" 2>/dev/null; then + kill "${pid}" 2>/dev/null || true + wait "${pid}" 2>/dev/null || true + fi + } + trap cleanup EXIT + + PORT="${port}" "${binary}" >"${log_file}" 2>&1 & + pid=$! + + for _ in {1..30}; do + if ! kill -0 "${pid}" 2>/dev/null; then + wait "${pid}" 2>/dev/null || true + echo "SEA binary exited before becoming healthy." + cat "${log_file}" + exit 1 + fi + + if curl --silent --show-error --fail "http://127.0.0.1:${port}/healthz" >/dev/null; then + launched="true" + break + fi + + sleep 1 + done + + if [[ "${launched}" != "true" ]]; then + echo "SEA binary did not become healthy within 30 seconds." + cat "${log_file}" + exit 1 + fi + + - name: Upload artifact + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + if: ${{ inputs.upload-only-binary == '' || matrix.binary_name == inputs.upload-only-binary }} + with: + name: ${{ matrix.binary_name }} + # package.json is included to anchor artifact paths at repo root. + path: | + package.json + packages/server-v3/dist/sea/${{ matrix.binary_name }} + retention-days: 7 diff --git a/.github/workflows/stagehand-server-v4-release.yml b/.github/workflows/stagehand-server-v4-release.yml new file mode 100644 index 000000000..6d447eec3 --- /dev/null +++ b/.github/workflows/stagehand-server-v4-release.yml @@ -0,0 +1,326 @@ +name: Release stagehand/server-v4 + +on: + push: + branches: + - main + paths: + - .changeset/** + - .github/workflows/stagehand-server-v4-release.yml + workflow_dispatch: + +permissions: + contents: write + +concurrency: ${{ github.workflow }}-${{ github.ref }} + +env: + OAS_PATH: packages/server-v4/openapi.v4.yaml + +jobs: + detect: + name: Detect server-v4 release (changesets) + runs-on: ubuntu-latest + outputs: + release: ${{ steps.meta.outputs.release }} + version: ${{ steps.meta.outputs.version }} + tag: ${{ steps.meta.outputs.tag }} + steps: + - name: Checkout repository + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + fetch-depth: 1 + fetch-tags: true + + - uses: ./.github/actions/setup-node-pnpm-turbo + env: + PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: "1" + with: + use-prebuilt-artifacts: "false" + + - name: Determine release metadata + id: meta + shell: bash + run: | + set -euo pipefail + + remote_tags="$(mktemp)" + git ls-remote --tags --refs origin 'refs/tags/stagehand-server-v4/v*' \ + | awk '{sub("refs/tags/", "", $2); print $2}' \ + > "${remote_tags}" + + latest_tag_file="$(mktemp)" + REMOTE_TAGS_FILE="${remote_tags}" LATEST_TAG_FILE="${latest_tag_file}" node <<'NODE' + const fs = require('fs'); + + const tags = fs + .readFileSync(process.env.REMOTE_TAGS_FILE, 'utf8') + .split('\n') + .map((line) => line.trim()) + .filter(Boolean); + + const parseTag = (tag) => { + const match = /^stagehand-server-v4\/v(\d+)\.(\d+)\.(\d+)$/.exec(tag); + if (!match) { + return null; + } + return match.slice(1).map((part) => Number(part)); + }; + + const compareVersions = (left, right) => { + for (let i = 0; i < 3; i += 1) { + if (left[i] !== right[i]) { + return left[i] - right[i]; + } + } + return 0; + }; + + let best = null; + for (const tag of tags) { + const version = parseTag(tag); + if (!version) { + continue; + } + if (best === null || compareVersions(version, best.version) > 0) { + best = { tag, version }; + } + } + + fs.writeFileSync(process.env.LATEST_TAG_FILE, best?.tag ?? '', 'utf8'); + NODE + latest_tag="$(cat "${latest_tag_file}")" + + changed_files="$(mktemp)" + if [ -n "${latest_tag}" ]; then + git fetch --force origin "refs/tags/${latest_tag}:refs/tags/${latest_tag}" + git diff --diff-filter=d --name-only "${latest_tag}"..HEAD -- '.changeset/*.md' > "${changed_files}" + else + find .changeset -maxdepth 1 -name '*.md' -print | sort > "${changed_files}" + fi + + LATEST_TAG="${latest_tag}" REMOTE_TAGS_FILE="${remote_tags}" CHANGED_CHANGESET_FILES="${changed_files}" node <<'NODE' + const fs = require('fs'); + const path = require('path'); + + const packageName = '@browserbasehq/stagehand-server-v4'; + const latestTag = process.env.LATEST_TAG || ''; + const remoteTags = fs + .readFileSync(process.env.REMOTE_TAGS_FILE, 'utf8') + .split('\n') + .map((line) => line.trim()) + .filter(Boolean); + const changesetFiles = fs + .readFileSync(process.env.CHANGED_CHANGESET_FILES, 'utf8') + .split('\n') + .map((line) => line.trim()) + .filter(Boolean) + .filter((file) => path.basename(file) !== 'config.json'); + + const releasePriority = { patch: 0, minor: 1, major: 2 }; + const compareVersions = (left, right) => { + for (let i = 0; i < 3; i += 1) { + if (left[i] !== right[i]) { + return left[i] - right[i]; + } + } + return 0; + }; + const parseTag = (tag) => { + const match = /^stagehand-server-v4\/v(\d+)\.(\d+)\.(\d+)$/.exec(tag); + if (!match) { + return null; + } + return match.slice(1).map((part) => Number(part)); + }; + + let highestRemoteVersion = null; + for (const tag of remoteTags) { + const version = parseTag(tag); + if (!version) { + continue; + } + if (highestRemoteVersion === null || compareVersions(version, highestRemoteVersion) > 0) { + highestRemoteVersion = version; + } + } + + let highestReleaseType = null; + + for (const file of changesetFiles) { + const content = fs.readFileSync(file, 'utf8'); + const match = /^---\n([\s\S]*?)\n---/m.exec(content); + if (!match) { + continue; + } + + for (const rawLine of match[1].split('\n')) { + const line = rawLine.trim(); + if (!line || line.startsWith('#')) { + continue; + } + + const releaseMatch = /^"([^"]+)"\s*:\s*(patch|minor|major)\s*$/.exec(line); + if (!releaseMatch) { + continue; + } + + const [, releasePackage, releaseType] = releaseMatch; + if (releasePackage !== packageName) { + continue; + } + + if ( + highestReleaseType === null || + releasePriority[releaseType] > releasePriority[highestReleaseType] + ) { + highestReleaseType = releaseType; + } + } + } + + const shouldRelease = highestReleaseType !== null; + let version = ''; + + if (shouldRelease) { + if (!latestTag || highestRemoteVersion === null) { + throw new Error( + 'No existing remote stagehand-server-v4 release tag was found. Refusing to bootstrap from 0.0.0.', + ); + } + + const parts = [...highestRemoteVersion]; + if (highestReleaseType === 'major') { + parts[0] += 1; + parts[1] = 0; + parts[2] = 0; + } else if (highestReleaseType === 'minor') { + parts[1] += 1; + parts[2] = 0; + } else { + parts[2] += 1; + } + + if (compareVersions(parts, highestRemoteVersion) <= 0) { + throw new Error( + `Computed stagehand-server-v4 version ${parts.join('.')} does not advance remote latest ${highestRemoteVersion.join('.')}.`, + ); + } + + version = parts.join('.'); + } + + const release = shouldRelease ? 'true' : 'false'; + const tag = `stagehand-server-v4/v${version}`; + + const out = process.env.GITHUB_OUTPUT; + fs.appendFileSync(out, `release=${release}\n`); + fs.appendFileSync(out, `version=${version}\n`); + fs.appendFileSync(out, `tag=${tag}\n`); + NODE + + - name: Create stagehand/server-v4 tag + if: steps.meta.outputs.release == 'true' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + shell: bash + run: | + set -euo pipefail + + TAG="${{ steps.meta.outputs.tag }}" + VERSION="${{ steps.meta.outputs.version }}" + TARGET_SHA="${{ github.sha }}" + + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + + # Try to fetch the tag if it exists on remote; ignore failure for new tags + git fetch --force origin "refs/tags/${TAG}:refs/tags/${TAG}" 2>/dev/null || true + if git rev-parse -q --verify "refs/tags/${TAG}" >/dev/null; then + echo "Tag already exists: ${TAG}" + exit 0 + fi + + git tag -a "${TAG}" "${TARGET_SHA}" -m "stagehand/server-v4 v${VERSION}" + git push origin "${TAG}" + + build_binaries: + name: Build SEA binaries + needs: detect + if: needs.detect.outputs.release == 'true' + uses: ./.github/workflows/stagehand-server-v4-sea-build.yml + with: + matrix: | + [ + {"os":"ubuntu-latest","platform":"linux","arch":"x64","binary_name":"stagehand-server-v4-linux-x64","include_sourcemaps":false}, + {"os":"ubuntu-24.04-arm","platform":"linux","arch":"arm64","binary_name":"stagehand-server-v4-linux-arm64","include_sourcemaps":false}, + {"os":"macos-15","platform":"darwin","arch":"arm64","binary_name":"stagehand-server-v4-darwin-arm64","include_sourcemaps":false}, + {"os":"macos-15-intel","platform":"darwin","arch":"x64","binary_name":"stagehand-server-v4-darwin-x64","include_sourcemaps":false}, + {"os":"windows-latest","platform":"win32","arch":"x64","binary_name":"stagehand-server-v4-win32-x64.exe","include_sourcemaps":false}, + {"os":"windows-11-arm","platform":"win32","arch":"arm64","binary_name":"stagehand-server-v4-win32-arm64.exe","include_sourcemaps":false} + ] + + release: + name: Publish GitHub Release + needs: [detect, build_binaries] + if: needs.detect.outputs.release == 'true' + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + fetch-depth: 1 + fetch-tags: false + + - name: Prepare release assets directory + run: mkdir -p release-assets + + - name: Prepare stagehand/server-v4 release assets + run: | + set -euo pipefail + cp "${{ env.OAS_PATH }}" "release-assets/openapi.v4.stagehand-server-v4-${{ needs.detect.outputs.version }}.yaml" + + - name: Download SEA binary artifacts + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + pattern: stagehand-server-v4-* + path: . + merge-multiple: true + + - name: Collect SEA binaries + shell: bash + run: | + set -euo pipefail + shopt -s nullglob + for f in packages/server-v4/dist/sea/stagehand-server-v4-*; do + cp "$f" release-assets/ + done + + - name: Create checksums + shell: bash + run: | + set -euo pipefail + cd release-assets + # Only checksum binaries (exclude openapi yaml). Avoid failing if no matches. + shopt -s nullglob + files=(stagehand-server-v4-*) + bins=() + for f in "${files[@]}"; do + [[ "$f" == *openapi* ]] && continue + [[ -f "$f" ]] && bins+=("$f") + done + : > checksums.sha256 + if [ "${#bins[@]}" -gt 0 ]; then + shasum -a 256 "${bins[@]}" > checksums.sha256 + fi + + - name: Publish stagehand/server-v4 GitHub release + uses: softprops/action-gh-release@3bb12739c298aeb8a4eeaf626c5b8d85266b0e65 # v2.6.2 + with: + tag_name: ${{ needs.detect.outputs.tag }} + name: stagehand/server-v4 v${{ needs.detect.outputs.version }} + generate_release_notes: true + files: | + release-assets/openapi.v4.stagehand-server-v4-${{ needs.detect.outputs.version }}.yaml + release-assets/stagehand-server-v4-* + release-assets/checksums.sha256 diff --git a/.github/workflows/stagehand-server-v4-sea-build.yml b/.github/workflows/stagehand-server-v4-sea-build.yml new file mode 100644 index 000000000..a55bc76bf --- /dev/null +++ b/.github/workflows/stagehand-server-v4-sea-build.yml @@ -0,0 +1,180 @@ +name: Stagehand Server v4 SEA Build + +on: + workflow_call: + inputs: + matrix: + description: "JSON matrix include list for SEA binaries." + required: false + type: string + default: | + [ + {"os":"ubuntu-latest","platform":"linux","arch":"x64","binary_name":"stagehand-server-v4-linux-x64","include_sourcemaps":false}, + {"os":"ubuntu-24.04-arm","platform":"linux","arch":"arm64","binary_name":"stagehand-server-v4-linux-arm64","include_sourcemaps":false}, + {"os":"macos-15","platform":"darwin","arch":"arm64","binary_name":"stagehand-server-v4-darwin-arm64","include_sourcemaps":false}, + {"os":"macos-15-intel","platform":"darwin","arch":"x64","binary_name":"stagehand-server-v4-darwin-x64","include_sourcemaps":false}, + {"os":"windows-latest","platform":"win32","arch":"x64","binary_name":"stagehand-server-v4-win32-x64.exe","include_sourcemaps":false}, + {"os":"windows-11-arm","platform":"win32","arch":"arm64","binary_name":"stagehand-server-v4-win32-arm64.exe","include_sourcemaps":false} + ] + use-prebuilt-artifacts: + description: "Whether to download pre-built package artifacts." + required: false + type: string + default: "false" + restore-turbo-cache: + description: "Whether to restore local .turbo cache." + required: false + type: string + default: "true" + node-version: + description: "Node.js version for setup." + required: false + type: string + default: "20.x" + upload-only-binary: + description: "Upload only this binary (empty => upload all)." + required: false + type: string + default: "" + workflow_dispatch: + inputs: + matrix: + description: "JSON matrix include list for SEA binaries." + required: false + default: | + [ + {"os":"ubuntu-latest","platform":"linux","arch":"x64","binary_name":"stagehand-server-v4-linux-x64","include_sourcemaps":false}, + {"os":"ubuntu-24.04-arm","platform":"linux","arch":"arm64","binary_name":"stagehand-server-v4-linux-arm64","include_sourcemaps":false}, + {"os":"macos-15","platform":"darwin","arch":"arm64","binary_name":"stagehand-server-v4-darwin-arm64","include_sourcemaps":false}, + {"os":"macos-15-intel","platform":"darwin","arch":"x64","binary_name":"stagehand-server-v4-darwin-x64","include_sourcemaps":false}, + {"os":"windows-latest","platform":"win32","arch":"x64","binary_name":"stagehand-server-v4-win32-x64.exe","include_sourcemaps":false}, + {"os":"windows-11-arm","platform":"win32","arch":"arm64","binary_name":"stagehand-server-v4-win32-arm64.exe","include_sourcemaps":false} + ] + use-prebuilt-artifacts: + description: "Whether to download pre-built package artifacts." + required: false + type: string + default: "false" + restore-turbo-cache: + description: "Whether to restore local .turbo cache." + required: false + type: string + default: "true" + node-version: + description: "Node.js version for setup." + required: false + type: string + default: "20.x" + upload-only-binary: + description: "Upload only this binary (empty => upload all)." + required: false + type: string + default: "" + +jobs: + build_binaries: + name: Build SEA binaries (${{ matrix.binary_name }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJson(inputs.matrix) }} + + steps: + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + fetch-depth: 1 + fetch-tags: false + + - uses: ./.github/actions/setup-node-pnpm-turbo + env: + PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: "1" + PLAYWRIGHT_SKIP_DOWNLOAD: "1" + PUPPETEER_SKIP_DOWNLOAD: "1" + with: + use-prebuilt-artifacts: ${{ inputs.use-prebuilt-artifacts }} + restore-turbo-cache: ${{ inputs.restore-turbo-cache }} + node-version: ${{ inputs.node-version }} + + - name: Build SEA binary (ESM) + env: + SEA_TARGET_PLATFORM: ${{ matrix.platform }} + SEA_TARGET_ARCH: ${{ matrix.arch }} + SEA_BINARY_NAME: ${{ matrix.binary_name }} + SEA_INCLUDE_SOURCEMAPS: ${{ matrix.include_sourcemaps && '1' || '0' }} + run: pnpm exec turbo run build:sea:esm --filter=@browserbasehq/stagehand-server-v4 + + - name: Verify SEA binary exists + shell: bash + run: | + test -f "packages/server-v4/dist/sea/${{ matrix.binary_name }}" + + - name: Verify SEA binary launches cleanly + shell: bash + env: + RUNNER_ARCH: ${{ runner.arch }} + run: | + set -euo pipefail + + binary="packages/server-v4/dist/sea/${{ matrix.binary_name }}" + matrix_arch="${{ matrix.arch }}" + runner_arch="$(echo "${RUNNER_ARCH}" | tr '[:upper:]' '[:lower:]')" + + if [[ "${matrix_arch}" != "${runner_arch}" ]]; then + echo "Runner arch (${runner_arch}) does not match matrix arch (${matrix_arch})." + echo "Launch verification must run on same-arch runners." + exit 1 + fi + + if [[ "${{ matrix.platform }}" != "win32" ]]; then + chmod +x "${binary}" + fi + + port="$((30000 + RANDOM % 10000))" + log_file="$(mktemp)" + launched="false" + + cleanup() { + if [[ -n "${pid:-}" ]] && kill -0 "${pid}" 2>/dev/null; then + kill "${pid}" 2>/dev/null || true + wait "${pid}" 2>/dev/null || true + fi + } + trap cleanup EXIT + + PORT="${port}" "${binary}" >"${log_file}" 2>&1 & + pid=$! + + for _ in {1..30}; do + if ! kill -0 "${pid}" 2>/dev/null; then + wait "${pid}" 2>/dev/null || true + echo "SEA binary exited before becoming healthy." + cat "${log_file}" + exit 1 + fi + + if curl --silent --show-error --fail "http://127.0.0.1:${port}/healthz" >/dev/null; then + launched="true" + break + fi + + sleep 1 + done + + if [[ "${launched}" != "true" ]]; then + echo "SEA binary did not become healthy within 30 seconds." + cat "${log_file}" + exit 1 + fi + + - name: Upload artifact + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + if: ${{ inputs.upload-only-binary == '' || matrix.binary_name == inputs.upload-only-binary }} + with: + name: ${{ matrix.binary_name }} + # package.json is included to anchor artifact paths at repo root. + path: | + package.json + packages/server-v4/dist/sea/${{ matrix.binary_name }} + retention-days: 7 diff --git a/.github/workflows/stainless.yml b/.github/workflows/stainless.yml new file mode 100644 index 000000000..e271a5275 --- /dev/null +++ b/.github/workflows/stainless.yml @@ -0,0 +1,61 @@ +name: Build SDKs for pull request + +on: + pull_request: + types: + - opened + - synchronize + - reopened + - closed + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number }} + cancel-in-progress: true + +env: + STAINLESS_ORG: ${{ vars.STAINLESS_ORG }} + STAINLESS_PROJECT: ${{ vars.STAINLESS_PROJECT }} + OAS_PATH: packages/server-v3/openapi.v3.yaml + +jobs: + preview: + # Fork PRs don't receive repo vars/secrets, so the Stainless action can only fail there + if: github.event.action != 'closed' && github.event.pull_request.head.repo.full_name == github.repository + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: write + steps: + - name: Checkout repository + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + fetch-depth: 2 + + - name: Run preview builds + uses: stainless-api/upload-openapi-spec-action/preview@fd70a2b6f380b00cad97b74ab14ab74918f96a65 # v1.13.3 + with: + stainless_api_key: ${{ secrets.STAINLESS_API_KEY }} + org: ${{ env.STAINLESS_ORG }} + project: ${{ env.STAINLESS_PROJECT }} + oas_path: ${{ env.OAS_PATH }} + config_path: stainless.yml + + merge: + if: github.event.action == 'closed' && github.event.pull_request.merged == true && github.event.pull_request.base.ref == 'main' && github.event.pull_request.head.repo.full_name == github.repository + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: write + steps: + - name: Checkout repository + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + fetch-depth: 2 + - name: Run merge build + uses: stainless-api/upload-openapi-spec-action/merge@fd70a2b6f380b00cad97b74ab14ab74918f96a65 # v1.13.3 + with: + stainless_api_key: ${{ secrets.STAINLESS_API_KEY }} + org: ${{ env.STAINLESS_ORG }} + project: ${{ env.STAINLESS_PROJECT }} + oas_path: ${{ env.OAS_PATH }} + config_path: stainless.yml diff --git a/.gitignore b/.gitignore index e5ea06bbe..5fc5921f7 100644 --- a/.gitignore +++ b/.gitignore @@ -9,12 +9,32 @@ screenshot.png .env downloads/ dist/ -evals/**/public -lib/dom/build/ -evals/public +.browserbase/ +packages/evals/**/public +packages/core/lib/dom/build/ +packages/core/lib/v3/dom/build/ +packages/evals/public *.tgz evals/playground.ts tmp/ eval-summary.json -pnpm-lock.yaml +package-lock.json evals/deterministic/tests/BrowserContext/tmp-test.har +packages/core/lib/version.ts +packages/core/test-results/ +/examples/inference_summary +/inference_summary +.turbo +.idea +coverage/ +ctrf/ +.stagehand-sea/ +.playwright*/ +**/.playwright*/ +packages/evals/playwright-mcp-screenshot-*.png +packages/evals/chrome-devtools-mcp-screenshot-*.png +.trajectories/ + +# The browse CLI has a command source dir named "downloads" that must be tracked +# despite the broad downloads/ rule above. +!packages/cli/src/commands/cloud/sessions/downloads/ diff --git a/.husky/pre-commit b/.husky/pre-commit new file mode 100644 index 000000000..5ee7abd87 --- /dev/null +++ b/.husky/pre-commit @@ -0,0 +1 @@ +pnpm exec lint-staged diff --git a/.prettierignore b/.prettierignore index 9581fb07d..b8cd807b6 100644 --- a/.prettierignore +++ b/.prettierignore @@ -1,3 +1,22 @@ pnpm-lock.yaml README.md -**/*.json \ No newline at end of file +**/*.json +docs/ +.github/ +dist/ +node_modules/ +lib/dom/build/ +lib/v3/dom/build/ +packages/core/dist/ +packages/core/lib/dom/build/ +packages/core/lib/v3/dom/build/ +packages/cli/dist/ +packages/evals/dist/ +packages/docs/ +*.min.js +.browserbase/ +.browserbase/** +**/.browserbase/ +**/.browserbase/** +stainless.yml +openapi.*.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index 2b34a6abc..b37ce4d87 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,222 @@ # @browserbasehq/stagehand +## 3.0.0 + +### Major Changes + +- Removes internal Playwright dependency +- A generous 20-40% speed increase across `act`, `extract`, & `observe` calls +- Compatibility with Playwright, Puppeteer, and Patchright +- Automatic action caching (agent, stagehand.act). Go from CUA → deterministic scripts w/o inference +- A suite of non AI primitives: + - `page` + - `locator` (built in closed mode shadow root traversal, with xpaths & css selectors) + - `frameLocator` + - `deepLocator` (crosses iframes & shadow roots) +- bun compatibility +- Simplified extract schemas +- CSS selector support (id-based support coming soon) +- Targeted extract and observe across iframes & shadow roots +- More intuitive type names (observeResult is now action, act accepts an instruction string instead of an action string, solidified ModelConfiguration) + +Check the [migration guide](https://docs.stagehand.dev/v3/migrations/v2) for more information + +## 2.5.0 + +### Minor Changes + +- [#981](https://github.com/browserbase/stagehand/pull/981) [`8244ab2`](https://github.com/browserbase/stagehand/commit/8244ab247cd679962685ae2f7c54e874ce1fa614) Thanks [@sameelarif](https://github.com/sameelarif)! - Added support for `stagehand.agent` to interact with MCP servers as well as custom tools to be passed in. For more information, reference the [MCP integrations documentation](https://docs.stagehand.dev/best-practices/mcp-integrations) + +### Patch Changes + +- [#959](https://github.com/browserbase/stagehand/pull/959) [`09b5e1e`](https://github.com/browserbase/stagehand/commit/09b5e1e9c23c845903686db6665cc968ac34efbb) Thanks [@filip-michalsky](https://github.com/filip-michalsky)! - add webvoyager evals + +- [#1049](https://github.com/browserbase/stagehand/pull/1049) [`e3734b9`](https://github.com/browserbase/stagehand/commit/e3734b9c98352d5f0a4eca49791b0bbf2130ab41) Thanks [@miguelg719](https://github.com/miguelg719)! - Support local MCP server connections + +- [#1025](https://github.com/browserbase/stagehand/pull/1025) [`be85b19`](https://github.com/browserbase/stagehand/commit/be85b19679a826f19702e00f0aae72fce1118ec8) Thanks [@tkattkat](https://github.com/tkattkat)! - add support for custom baseUrl within openai provider + +- [#1040](https://github.com/browserbase/stagehand/pull/1040) [`88d1565`](https://github.com/browserbase/stagehand/commit/88d1565c65bb65a104fea2d5f5e862bbbda69677) Thanks [@miguelg719](https://github.com/miguelg719)! - Allow OpenAI CUA to take in an optional baseURL + +- [#1046](https://github.com/browserbase/stagehand/pull/1046) [`ab5d6ed`](https://github.com/browserbase/stagehand/commit/ab5d6ede19aabc059badc4247f1cb2c6c9e71bae) Thanks [@tkattkat](https://github.com/tkattkat)! - Add support for gpt-5 in operator agent + +## 2.4.4 + +### Patch Changes + +- [#1012](https://github.com/browserbase/stagehand/pull/1012) [`9e8c173`](https://github.com/browserbase/stagehand/commit/9e8c17374fdc8fbe7f26e6cf802c36bd14f11039) Thanks [@miguelg719](https://github.com/miguelg719)! - Fix disabling api validation whenever a customLLM client is provided + +## 2.4.3 + +### Patch Changes + +- [#951](https://github.com/browserbase/stagehand/pull/951) [`f45afdc`](https://github.com/browserbase/stagehand/commit/f45afdccc8680650755fee66ffbeac32b41e075d) Thanks [@miguelg719](https://github.com/miguelg719)! - Patch GPT-5 new api format + +- [#954](https://github.com/browserbase/stagehand/pull/954) [`261bba4`](https://github.com/browserbase/stagehand/commit/261bba43fa79ac3af95328e673ef3e9fced3279b) Thanks [@seanmcguire12](https://github.com/seanmcguire12)! - add support for shadow DOMs (open & closed mode) when experimental: true + +- [#944](https://github.com/browserbase/stagehand/pull/944) [`8de7bd8`](https://github.com/browserbase/stagehand/commit/8de7bd8635c2051cd8025e365c6c8aa83d81c7e7) Thanks [@seanmcguire12](https://github.com/seanmcguire12)! - Bump zod version compatibility and add pathing spec + +- [#919](https://github.com/browserbase/stagehand/pull/919) [`3d80421`](https://github.com/browserbase/stagehand/commit/3d804210a106a6828c7fa50f8b765b10afd4cc6a) Thanks [@seanmcguire12](https://github.com/seanmcguire12)! - enable scrolling inside of iframes + +- [#963](https://github.com/browserbase/stagehand/pull/963) [`0ead63d`](https://github.com/browserbase/stagehand/commit/0ead63d6526f6c286362b74b6407c8bebc900e69) Thanks [@tkattkat](https://github.com/tkattkat)! - Properly handle images in evaluator + clean up response parsing logic + +- [#961](https://github.com/browserbase/stagehand/pull/961) [`8422828`](https://github.com/browserbase/stagehand/commit/8422828c4cd5fd5ebcf348cfbdb40c768bb76dd9) Thanks [@tkattkat](https://github.com/tkattkat)! - Add more evals for stagehand agent + +- [#946](https://github.com/browserbase/stagehand/pull/946) [`b769206`](https://github.com/browserbase/stagehand/commit/b7692060f98a2f49aeeefb90d8789ed034b08ec2) Thanks [@seanmcguire12](https://github.com/seanmcguire12)! - fix: unable to act on/get content from some same process iframes + +- [#962](https://github.com/browserbase/stagehand/pull/962) [`72d2683`](https://github.com/browserbase/stagehand/commit/72d2683202af7e578d98367893964b33e0828de5) Thanks [@seanmcguire12](https://github.com/seanmcguire12)! - handle namespaced elements in xpath build step + +## 2.4.2 + +### Patch Changes + +- [#865](https://github.com/browserbase/stagehand/pull/865) [`6b4e6e3`](https://github.com/browserbase/stagehand/commit/6b4e6e3f31d5496cf15728e9018eddeb04839542) Thanks [@seanmcguire12](https://github.com/seanmcguire12)! - improve type safety for trimTrailingTextNode + +- [#897](https://github.com/browserbase/stagehand/pull/897) [`e77d018`](https://github.com/browserbase/stagehand/commit/e77d0188683ebf596dfb78dfafbbca1dc32993f0) Thanks [@miguelg719](https://github.com/miguelg719)! - Fix selfHeal to remember intially received arguments + +- [#920](https://github.com/browserbase/stagehand/pull/920) [`c20adb9`](https://github.com/browserbase/stagehand/commit/c20adb95539fed8c56a4aa413262a9c65a8e6474) Thanks [@seanmcguire12](https://github.com/seanmcguire12)! - fix: tab handling on API + +- [#882](https://github.com/browserbase/stagehand/pull/882) [`b86df93`](https://github.com/browserbase/stagehand/commit/b86df93b9136aae96292121a29c25f3d74d84bf7) Thanks [@seanmcguire12](https://github.com/seanmcguire12)! - remove elements that don't have xpaths from observe response + +- [#905](https://github.com/browserbase/stagehand/pull/905) [`023c2c2`](https://github.com/browserbase/stagehand/commit/023c2c273b46d3792d7e5d3c902089487b16b531) Thanks [@tkattkat](https://github.com/tkattkat)! - Delete old images from anthropic cua client + +- [#925](https://github.com/browserbase/stagehand/pull/925) [`8c28647`](https://github.com/browserbase/stagehand/commit/8c2864755ecd05c8f7de235d4198deec0dd5f78e) Thanks [@miguelg719](https://github.com/miguelg719)! - Remove \_refreshPageFromApi() + +- [#887](https://github.com/browserbase/stagehand/pull/887) [`87e09c6`](https://github.com/browserbase/stagehand/commit/87e09c618940f364ec8af00455a19a17ec63cbd3) Thanks [@seanmcguire12](https://github.com/seanmcguire12)! - fix: allow xpaths with prepended 'xpath=' for targeted extract + +- [#864](https://github.com/browserbase/stagehand/pull/864) [`a611115`](https://github.com/browserbase/stagehand/commit/a61111525d70b450bdfc43f112380f44899c9e97) Thanks [@miguelg719](https://github.com/miguelg719)! - Temporarily patch custom clients serialization error on api + +- [#881](https://github.com/browserbase/stagehand/pull/881) [`69913fe`](https://github.com/browserbase/stagehand/commit/69913fe1dfb8201ae2aeffa5f049fb46ab02cbc2) Thanks [@miguelg719](https://github.com/miguelg719)! - Pass sdk version number to API for debugging + +- [#913](https://github.com/browserbase/stagehand/pull/913) [`b1b83a1`](https://github.com/browserbase/stagehand/commit/b1b83a1d334fe76e5f5f9dd32dc92c16b7d40ce6) Thanks [@seanmcguire12](https://github.com/seanmcguire12)! - move iframe out of 'experimental' + +- [#891](https://github.com/browserbase/stagehand/pull/891) [`be8497c`](https://github.com/browserbase/stagehand/commit/be8497cb6b142cc893cea9692b8c47bd19514c60) Thanks [@seanmcguire12](https://github.com/seanmcguire12)! - fix: nested iframe xpath bug + +- [#883](https://github.com/browserbase/stagehand/pull/883) [`98704c9`](https://github.com/browserbase/stagehand/commit/98704c9ed225ca25bbde4bb3dc286936e9c54471) Thanks [@seanmcguire12](https://github.com/seanmcguire12)! - add timeout for JS click + +- [#907](https://github.com/browserbase/stagehand/pull/907) [`04978bd`](https://github.com/browserbase/stagehand/commit/04978bdd30d2edcbc69eb9fd91358a16975ea2eb) Thanks [@seanmcguire12](https://github.com/seanmcguire12)! - store mapping of CDP frame ID -> page + +## 2.4.1 + +### Patch Changes + +- [#856](https://github.com/browserbase/stagehand/pull/856) [`8a43c5a`](https://github.com/browserbase/stagehand/commit/8a43c5a86d4da40cfaedd9cf2e42186928bdf946) Thanks [@seanmcguire12](https://github.com/seanmcguire12)! - set download behaviour by default + +- [#857](https://github.com/browserbase/stagehand/pull/857) [`890ffcc`](https://github.com/browserbase/stagehand/commit/890ffccac5e0a60ade64a46eb550c981ffb3e84a) Thanks [@miguelg719](https://github.com/miguelg719)! - return "not-supported" for elements inside the shadow-dom + +- [#844](https://github.com/browserbase/stagehand/pull/844) [`64c1072`](https://github.com/browserbase/stagehand/commit/64c10727bda50470483a3eb175c02842db0923a1) Thanks [@seanmcguire12](https://github.com/seanmcguire12)! - don't automatically close tabs + +- [#860](https://github.com/browserbase/stagehand/pull/860) [`b077d3f`](https://github.com/browserbase/stagehand/commit/b077d3f48a97f47a71ccc79ae39b41e7f07f9c04) Thanks [@miguelg719](https://github.com/miguelg719)! - Set default schema on extract options with no schema + +- [#842](https://github.com/browserbase/stagehand/pull/842) [`8bcb5d7`](https://github.com/browserbase/stagehand/commit/8bcb5d77debf6bf7601fd5c090efd7fde75c5d5e) Thanks [@seanmcguire12](https://github.com/seanmcguire12)! - improved handling for OS level dropdowns + +- [#846](https://github.com/browserbase/stagehand/pull/846) [`7bf10c5`](https://github.com/browserbase/stagehand/commit/7bf10c55b267078fe847c1d7f7a60d604f9c7c94) Thanks [@miguelg719](https://github.com/miguelg719)! - Filter attaching to target worker / shared_worker + +## 2.4.0 + +### Minor Changes + +- [#819](https://github.com/browserbase/stagehand/pull/819) [`6a18c1e`](https://github.com/browserbase/stagehand/commit/6a18c1ee1e46d55c6e90c4d5572e17ed8daa140c) Thanks [@seanmcguire12](https://github.com/seanmcguire12)! - try playwright click and fall back to JS click event + +### Patch Changes + +- [#826](https://github.com/browserbase/stagehand/pull/826) [`124e0d3`](https://github.com/browserbase/stagehand/commit/124e0d3bb54ddb6738ede6d7aa99a945ef1cacd1) Thanks [@seanmcguire12](https://github.com/seanmcguire12)! - fix issue where we are unable to take actions on text nodes + +- [#818](https://github.com/browserbase/stagehand/pull/818) [`1660751`](https://github.com/browserbase/stagehand/commit/1660751cd14cb5b27d44f8167216afb8d1c3c45c) Thanks [@miguelg719](https://github.com/miguelg719)! - Added CUA support for Claude 4 models + +- [#821](https://github.com/browserbase/stagehand/pull/821) [`cadac9d`](https://github.com/browserbase/stagehand/commit/cadac9da09123d12e5d496a0e8b12660964c1b33) Thanks [@seanmcguire12](https://github.com/seanmcguire12)! - use playwright instead of playwright test + +- [#832](https://github.com/browserbase/stagehand/pull/832) [`759da55`](https://github.com/browserbase/stagehand/commit/759da55775eb2df81d56ae18c0f386fd9b02a9f0) Thanks [@miguelg719](https://github.com/miguelg719)! - Fix \_refreshPageFromAPI to use parametrized apiKey + +- [#810](https://github.com/browserbase/stagehand/pull/810) [`a175a51`](https://github.com/browserbase/stagehand/commit/a175a519b8c14300db6f1ed30709e113d18e99db) Thanks [@miguelg719](https://github.com/miguelg719)! - Update logos + +- [#822](https://github.com/browserbase/stagehand/pull/822) [`8527a80`](https://github.com/browserbase/stagehand/commit/8527a80522c3eedb9516a6caa1a0e4e4be981a3d) Thanks [@miguelg719](https://github.com/miguelg719)! - Add model with date tag for OpenAI CUA + +- [#833](https://github.com/browserbase/stagehand/pull/833) [`55fca2f`](https://github.com/browserbase/stagehand/commit/55fca2f7da63cc0ef6e27b45a33f63c666cdce7e) Thanks [@seanmcguire12](https://github.com/seanmcguire12)! - adjust stagehandLogger.warn() level to be 1 instead of 0 + +## 2.3.1 + +### Patch Changes + +- [#796](https://github.com/browserbase/stagehand/pull/796) [`12a99b3`](https://github.com/browserbase/stagehand/commit/12a99b398d8a4c3eea3ca69a3cf793faaaf4aea3) Thanks [@miguelg719](https://github.com/miguelg719)! - Added a experimental flag to enable the newest and most experimental features + +- [#807](https://github.com/browserbase/stagehand/pull/807) [`2451797`](https://github.com/browserbase/stagehand/commit/2451797f64c0efa4a72fd70265110003c8d0a6cd) Thanks [@seanmcguire12](https://github.com/seanmcguire12)! - include version number in StagehandDefaultError message + +- [#803](https://github.com/browserbase/stagehand/pull/803) [`1d631a5`](https://github.com/browserbase/stagehand/commit/1d631a57a197390f672b718ae5199991ab27cfb1) Thanks [@miguelg719](https://github.com/miguelg719)! - Enable session affinity for cache optimization + +- [#804](https://github.com/browserbase/stagehand/pull/804) [`9c398bb`](https://github.com/browserbase/stagehand/commit/9c398bb9ec2d10bdb53ad5aa7e3b58cce24fdb2b) Thanks [@seanmcguire12](https://github.com/seanmcguire12)! - update operatorResponseSchema based on new openai spec + +- [#786](https://github.com/browserbase/stagehand/pull/786) [`c19ad7f`](https://github.com/browserbase/stagehand/commit/c19ad7f1e082e91fdeaa9c2ef63767a5a2b3a195) Thanks [@miguelg719](https://github.com/miguelg719)! - Handle reroute to account for rollout + +## 2.3.0 + +### Minor Changes + +- [#737](https://github.com/browserbase/stagehand/pull/737) [`6ef6073`](https://github.com/browserbase/stagehand/commit/6ef60730cab0ad9025f44b6eeb2c83751d1dcd35) Thanks [@seanmcguire12](https://github.com/seanmcguire12)! - deprecate useTextExtract and remove functionality + +### Patch Changes + +- [#741](https://github.com/browserbase/stagehand/pull/741) [`5680d25`](https://github.com/browserbase/stagehand/commit/5680d2509352c383ad502c9f4fabde01fa638833) Thanks [@seanmcguire12](https://github.com/seanmcguire12)! - use safeparse for zod validation + +- [#783](https://github.com/browserbase/stagehand/pull/783) [`4de92a8`](https://github.com/browserbase/stagehand/commit/4de92a8af461fc95063faf39feee1d49259f58ba) Thanks [@miguelg719](https://github.com/miguelg719)! - Fix the readme logo link + +## 2.2.1 + +### Patch Changes + +- [#721](https://github.com/browserbase/stagehand/pull/721) [`be8652e`](https://github.com/browserbase/stagehand/commit/be8652e770b57fdb3299fa0b2efa4eb0e816434e) Thanks [@miguelg719](https://github.com/miguelg719)! - Fix stagehand.close() functionality to include calling browser.close() + +- [#724](https://github.com/browserbase/stagehand/pull/724) [`6b413b7`](https://github.com/browserbase/stagehand/commit/6b413b7ad00b13ca0bd53ee2e7393023821408b6) Thanks [@seanmcguire12](https://github.com/seanmcguire12)! - rm refine step in extract + +- [#712](https://github.com/browserbase/stagehand/pull/712) [`7eafbd9`](https://github.com/browserbase/stagehand/commit/7eafbd9b1a73b37effa444929767df7c592caf02) Thanks [@seanmcguire12](https://github.com/seanmcguire12)! - deprecated `onlyVisible` param and remove its functionality + +- [#725](https://github.com/browserbase/stagehand/pull/725) [`1b50aa6`](https://github.com/browserbase/stagehand/commit/1b50aa61cf0a429dd6cb2760a08f7f698a50454b) Thanks [@seanmcguire12](https://github.com/seanmcguire12)! - dont overwrite .describe() when user defines a zod schema with z.string().url().describe() + +- [#717](https://github.com/browserbase/stagehand/pull/717) [`f2b7f1f`](https://github.com/browserbase/stagehand/commit/f2b7f1f284eef1f96753319b66c7d0b273a6f8cd) Thanks [@seanmcguire12](https://github.com/seanmcguire12)! - don't publish uncompiled ts to npm + +- [#719](https://github.com/browserbase/stagehand/pull/719) [`c8d672f`](https://github.com/browserbase/stagehand/commit/c8d672f7c410c256defbc2e87ead99239837aa28) Thanks [@seanmcguire12](https://github.com/seanmcguire12)! - fix `Invalid schema for response_format` error when extracting links + +- [#722](https://github.com/browserbase/stagehand/pull/722) [`bebf204`](https://github.com/browserbase/stagehand/commit/bebf2044502333c694743078c5b0c9deae11fb79) Thanks [@seanmcguire12](https://github.com/seanmcguire12)! - replace NBSP with regular space & remove special characters from dom+a11y tree + +- [#714](https://github.com/browserbase/stagehand/pull/714) [`37d6810`](https://github.com/browserbase/stagehand/commit/37d6810a704773d0383a86f98f5f17c7d5b21975) Thanks [@miguelg719](https://github.com/miguelg719)! - Fix the native AI SDK client implementation to optionally take in an API key + +## 2.2.0 + +### Minor Changes + +- [#655](https://github.com/browserbase/stagehand/pull/655) [`8814af9`](https://github.com/browserbase/stagehand/commit/8814af9ece99fddc3dd9fb32671d0513a3a00c67) Thanks [@seanmcguire12](https://github.com/seanmcguire12)! - extract links + +- [#675](https://github.com/browserbase/stagehand/pull/675) [`35c55eb`](https://github.com/browserbase/stagehand/commit/35c55ebf6c2867801a0a6f6988a883c8cb90cf9a) Thanks [@tkattkat](https://github.com/tkattkat)! - Added Gemini 2.5 Flash to Google supported models + +- [#668](https://github.com/browserbase/stagehand/pull/668) [`5c6d2cf`](https://github.com/browserbase/stagehand/commit/5c6d2cf89c9fbf198485506ed9ed75e07aec5cd4) Thanks [@miguelg719](https://github.com/miguelg719)! - Added a new class - Stagehand Evaluator - that wraps around a Stagehand object to determine whether a task is successful or not. Currently used for agent evals + +### Patch Changes + +- [#706](https://github.com/browserbase/stagehand/pull/706) [`18ac6fb`](https://github.com/browserbase/stagehand/commit/18ac6fba30f45b7557cecb890f4e84c75de8383c) Thanks [@seanmcguire12](https://github.com/seanmcguire12)! - remove unused fillInVariables fn + +- [#692](https://github.com/browserbase/stagehand/pull/692) [`6b95248`](https://github.com/browserbase/stagehand/commit/6b95248d6e02e5304ce4dd60499e31fc42af57eb) Thanks [@miguelg719](https://github.com/miguelg719)! - Updated the list of OpenAI models (4.1, o3...) + +- [#688](https://github.com/browserbase/stagehand/pull/688) [`7d81b3c`](https://github.com/browserbase/stagehand/commit/7d81b3c951c1f3dfc46845aefcc26ff175299bca) Thanks [@seanmcguire12](https://github.com/seanmcguire12)! - wrap page.evaluate to make sure we have injected browser side scripts before calling them + +- [#664](https://github.com/browserbase/stagehand/pull/664) [`b5ca00a`](https://github.com/browserbase/stagehand/commit/b5ca00a25ad0c33a5f4d3198e1bc59edb9956e7c) Thanks [@miguelg719](https://github.com/miguelg719)! - remove unnecessary log + +- [#683](https://github.com/browserbase/stagehand/pull/683) [`8f0f97b`](https://github.com/browserbase/stagehand/commit/8f0f97bc491e23ff0078c802aaf509fd04173c37) Thanks [@seanmcguire12](https://github.com/seanmcguire12)! - use javsacript click instead of playwright + +- [#705](https://github.com/browserbase/stagehand/pull/705) [`346ef5d`](https://github.com/browserbase/stagehand/commit/346ef5d0132dc1418dac18d26640a8df0435af57) Thanks [@miguelg719](https://github.com/miguelg719)! - Fixed removing a hanging observation map that is no longer used + +- [#698](https://github.com/browserbase/stagehand/pull/698) [`c145bc1`](https://github.com/browserbase/stagehand/commit/c145bc1d90ffd0d71c412de3af1c26c121e0b101) Thanks [@sameelarif](https://github.com/sameelarif)! - Fixing LLM client support to natively integrate with AI SDK + +- [#687](https://github.com/browserbase/stagehand/pull/687) [`edd6d3f`](https://github.com/browserbase/stagehand/commit/edd6d3feb47aac9f312a5edad78bf850ae1541db) Thanks [@miguelg719](https://github.com/miguelg719)! - Fixed the schema input for Gemini's response model + +- [#678](https://github.com/browserbase/stagehand/pull/678) [`5ec43d8`](https://github.com/browserbase/stagehand/commit/5ec43d8b9568c0f86b3e24bd83d1826c837656ed) Thanks [@seanmcguire12](https://github.com/seanmcguire12)! - allow form filling when form is not top-most element + +- [#694](https://github.com/browserbase/stagehand/pull/694) [`b8cc164`](https://github.com/browserbase/stagehand/commit/b8cc16405b712064a54c8cd591750368a47f35ea) Thanks [@seanmcguire12](https://github.com/seanmcguire12)! - add telemetry for cua agents to stagehand.metrics + +- [#699](https://github.com/browserbase/stagehand/pull/699) [`d9f4243`](https://github.com/browserbase/stagehand/commit/d9f4243f6a8c8d4f3003ad6589f7eb4da6d23d0f) Thanks [@seanmcguire12](https://github.com/seanmcguire12)! - rm deprecated primitives from stagehand object + +- [#710](https://github.com/browserbase/stagehand/pull/710) [`9f4ab76`](https://github.com/browserbase/stagehand/commit/9f4ab76a0c1f0c2171290765c48c3bcea5b50e0f) Thanks [@seanmcguire12](https://github.com/seanmcguire12)! - support targeted extract for domExtract + +- [#677](https://github.com/browserbase/stagehand/pull/677) [`bc5a731`](https://github.com/browserbase/stagehand/commit/bc5a731241f7f4c5040dd672d8e3787555766421) Thanks [@miguelg719](https://github.com/miguelg719)! - Fixes a redundant unnecessary log + ## 2.1.0 ### Minor Changes diff --git a/README.md b/README.md index 788d6c073..5122826fb 100644 --- a/README.md +++ b/README.md @@ -1,30 +1,29 @@ -
-
    + -

    - The production-ready framework for AI browser automations.
    + The AI Browser Automation Framework
    Read the Docs

    - - MIT License + + MIT License - + - - Slack Community + + Discord Community

    @@ -33,81 +32,78 @@ browserbase%2Fstagehand | Trendshift

    +

    + + Ask DeepWiki + +

    + +

    +If you're looking for the Python implementation, you can find it + here +

    + +## What is Stagehand? + +Stagehand is a browser automation framework used to control web browsers with natural language and code. By combining the power of AI with the precision of code, Stagehand makes web automation flexible, maintainable, and actually reliable. + ## Why Stagehand? -Most existing browser automation tools either require you to write low-level code in a framework like Selenium, Playwright, or Puppeteer, or use high-level agents that can be unpredictable in production. By letting developers choose what to write in code vs. natural language, Stagehand is the natural choice for browser automations in production. +Most existing browser automation tools either require you to write low-level code in a framework like Selenium, Playwright, or Puppeteer, or use high-level agents that can be unpredictable in production. By letting developers choose what to write in code vs. natural language (and bridging the gap between the two) Stagehand is the natural choice for browser automations in production. -1. **Choose when to write code vs. natural language**: use AI when you want to navigate unfamiliar pages, and use code ([Playwright](https://playwright.dev/)) when you know exactly what you want to do. +1. **Choose when to write code vs. natural language**: use AI when you want to navigate unfamiliar pages, and use code when you know exactly what you want to do. -2. **Preview and cache actions**: Stagehand lets you preview AI actions before running them, and also helps you easily cache repeatable actions to save time and tokens. +2. **Go from AI-driven to repeatable workflows**: Stagehand lets you preview AI actions before running them, and also helps you easily cache repeatable actions to save time and tokens. -3. **Computer use models with one line of code**: Stagehand lets you integrate SOTA computer use models from OpenAI and Anthropic into the browser with one line of code. +3. **Write once, run forever**: Stagehand's auto-caching combined with self-healing remembers previous actions, runs without LLM inference, and knows when to involve AI whenever the website changes and your automation breaks. + +## Getting Started + +Start with Stagehand with one line of code, or check out our [Quickstart Guide](https://docs.stagehand.dev/v3/first-steps/quickstart) for more information: + +```bash +npx create-browser-app +``` ## Example Here's how to build a sample browser automation with Stagehand: -
    -
    - See Stagehand in Action -
    -
    - ```typescript -// Use Playwright functions on the page object -const page = stagehand.page; +// Stagehand's CDP engine provides an optimized, low level interface to the browser built for automation +const page = stagehand.context.pages()[0]; await page.goto("https://github.com/browserbase"); // Use act() to execute individual actions -await page.act("click on the stagehand repo"); +await stagehand.act("click on the stagehand repo"); -// Use Computer Use agents for larger actions -const agent = stagehand.agent({ - provider: "openai", - model: "computer-use-preview", -}); +// Use agent() for multi-step tasks +const agent = stagehand.agent(); await agent.execute("Get to the latest PR"); -// Use extract() to read data from the page -const { author, title } = await page.extract({ - instruction: "extract the author and title of the PR", - schema: z.object({ +// Use extract() to get structured data from the page +const { author, title } = await stagehand.extract( + "extract the author and title of the PR", + z.object({ author: z.string().describe("The username of the PR author"), title: z.string().describe("The title of the PR"), }), -}); +); ``` ## Documentation Visit [docs.stagehand.dev](https://docs.stagehand.dev) to view the full documentation. -## Getting Started - -Start with Stagehand with one line of code, or check out our [Quickstart Guide](https://docs.stagehand.dev/get_started/quickstart) for more information: - -```bash -npx create-browser-app -``` - - ### Build and Run from Source ```bash git clone https://github.com/browserbase/stagehand.git cd stagehand -npm install -npx playwright install -npm run build -npm run example # run the blank script at ./examples/example.ts +pnpm install +pnpm run build +pnpm run example # run the blank script at ./examples/example.ts ``` Stagehand is best when you have an API key for an LLM provider and Browserbase credentials. To add these to your project, run: @@ -117,26 +113,40 @@ cp .env.example .env nano .env # Edit the .env file to add API keys ``` +### Installing from a branch + +To install Stagehand directly from a GitHub branch, install the core package subdirectory: + +```bash +pnpm add "github:browserbase/stagehand#&path:/packages/core" +``` + +Or set it in your project's `package.json`: + +```json +"@browserbasehq/stagehand": "github:browserbase/stagehand#&path:/packages/core" +``` + ## Contributing -> [!NOTE] -> We highly value contributions to Stagehand! For questions or support, please join our [Slack community](https://stagehand.dev/slack). +> [!NOTE] +> We highly value contributions to Stagehand! For questions or support, please join our [Discord community](https://stagehand.dev/discord). -At a high level, we're focused on improving reliability, speed, and cost in that order of priority. If you're interested in contributing, we strongly recommend reaching out to [Anirudh Kamath](https://x.com/kamathematic) or [Paul Klein](https://x.com/pk_iv) in our [Slack community](https://stagehand.dev/slack) before starting to ensure that your contribution aligns with our goals. +At a high level, we're focused on improving reliability, extensibility, speed, and cost in that order of priority. If you're interested in contributing, **bug fixes and small improvements are the best way to get started**. For more involved features, we strongly recommend reaching out to [Miguel Gonzalez](https://x.com/miguel_gonzf) or [Paul Klein](https://x.com/pk_iv) in our [Discord community](https://stagehand.dev/discord) before starting to ensure that your contribution aligns with our goals. -For more information, please see our [Contributing Guide](https://docs.stagehand.dev/contributions/contributing). -## Acknowledgements + -This project heavily relies on [Playwright](https://playwright.dev/) as a resilient backbone to automate the web. It also would not be possible without the awesome techniques and discoveries made by [tarsier](https://github.com/reworkd/tarsier), and [fuji-web](https://github.com/normal-computing/fuji-web). +## Acknowledgements We'd like to thank the following people for their major contributions to Stagehand: - [Paul Klein](https://github.com/pkiv) -- [Anirudh Kamath](https://github.com/kamath) - [Sean McGuire](https://github.com/seanmcguire12) - [Miguel Gonzalez](https://github.com/miguelg719) - [Sameel Arif](https://github.com/sameelarif) +- [Thomas Katwan](https://github.com/tkattkat) - [Filip Michalsky](https://github.com/filip-michalsky) +- [Anirudh Kamath](https://github.com/kamath) - [Jeremy Press](https://x.com/jeremypress) - [Navid Pour](https://github.com/navidpour) diff --git a/claude.md b/claude.md new file mode 100644 index 000000000..d66f98b52 --- /dev/null +++ b/claude.md @@ -0,0 +1,299 @@ +# Stagehand Project + +This is a project that uses Stagehand V3, a browser automation framework with AI-powered `act`, `extract`, `observe`, and `agent` methods. + +The main class can be imported as `Stagehand` from `@browserbasehq/stagehand`. + +**Key Classes:** + +- `Stagehand`: Main orchestrator class providing `act`, `extract`, `observe`, and `agent` methods +- `context`: A `V3Context` object that manages browser contexts and pages +- `page`: Individual page objects accessed via `stagehand.context.pages()[i]` or created with `stagehand.context.newPage()` + +## Initialize + +```typescript +import { Stagehand } from "@browserbasehq/stagehand"; + +const stagehand = new Stagehand({ + env: "LOCAL", // or "BROWSERBASE" + verbose: 2, // 0, 1, or 2 + model: "openai/gpt-4.1-mini", // or any supported model +}); + +await stagehand.init(); + +// Access the browser context and pages +const page = stagehand.context.pages()[0]; +const context = stagehand.context; + +// Create new pages if needed +const page2 = await stagehand.context.newPage(); +``` + +## Act + +Actions are called on the `stagehand` instance (not the page). Use atomic, specific instructions: + +```typescript +// Act on the current active page +await stagehand.act("click the sign in button"); + +// Act on a specific page (when you need to target a page that isn't currently active) +await stagehand.act("click the sign in button", { page: page2 }); +``` + +**Important:** Act instructions should be atomic and specific: + +- ✅ Good: "Click the sign in button" or "Type 'hello' into the search input" +- ❌ Bad: "Order me pizza" or "Type in the search bar and hit enter" (multi-step) + +### Observe + Act Pattern (Recommended) + +Cache the results of `observe` to avoid unexpected DOM changes: + +```typescript +const instruction = "Click the sign in button"; + +// Get candidate actions +const actions = await stagehand.observe(instruction); + +// Execute the first action +await stagehand.act(actions[0]); +``` + +To target a specific page: + +```typescript +const actions = await stagehand.observe("select blue as the favorite color", { + page: page2, +}); +await stagehand.act(actions[0], { page: page2 }); +``` + +## Extract + +Extract data from pages using natural language instructions. The `extract` method is called on the `stagehand` instance. + +### Basic Extraction (with schema) + +```typescript +import { z } from "zod"; + +// Extract with explicit schema +const data = await stagehand.extract( + "extract all apartment listings with prices and addresses", + z.object({ + listings: z.array( + z.object({ + price: z.string(), + address: z.string(), + }), + ), + }), +); + +console.log(data.listings); +``` + +### Simple Extraction (without schema) + +```typescript +// Extract returns a default object with 'extraction' field +const result = await stagehand.extract("extract the sign in button text"); + +console.log(result); +// Output: { extraction: "Sign in" } + +// Or destructure directly +const { extraction } = await stagehand.extract( + "extract the sign in button text", +); +console.log(extraction); // "Sign in" +``` + +### Targeted Extraction + +Extract data from a specific element using a selector: + +```typescript +const reason = await stagehand.extract( + "extract the reason why script injection fails", + z.string(), + { selector: "/html/body/div[2]/div[3]/iframe/html/body/p[2]" }, +); +``` + +### URL Extraction + +When extracting links or URLs, use `z.string().url()`: + +```typescript +const { links } = await stagehand.extract( + "extract all navigation links", + z.object({ + links: z.array(z.string().url()), + }), +); +``` + +### Extracting from a Specific Page + +```typescript +// Extract from a specific page (when you need to target a page that isn't currently active) +const data = await stagehand.extract( + "extract the placeholder text on the name field", + { page: page2 }, +); +``` + +## Observe + +Plan actions before executing them. Returns an array of candidate actions: + +```typescript +// Get candidate actions on the current active page +const [action] = await stagehand.observe("Click the sign in button"); + +// Execute the action +await stagehand.act(action); +``` + +Observing on a specific page: + +```typescript +// Target a specific page (when you need to target a page that isn't currently active) +const actions = await stagehand.observe("find the next page button", { + page: page2, +}); +await stagehand.act(actions[0], { page: page2 }); +``` + +## Agent + +Use the `agent` method to autonomously execute complex, multi-step tasks. + +### Basic Agent Usage + +```typescript +const page = stagehand.context.pages()[0]; +await page.goto("https://www.google.com"); + +const agent = stagehand.agent({ + model: "google/gemini-2.0-flash", + executionModel: "google/gemini-2.0-flash", +}); + +const result = await agent.execute({ + instruction: "Search for the stock price of NVDA", + maxSteps: 20, +}); + +console.log(result.message); +``` + +### Computer Use Agent (CUA) + +For more advanced scenarios using computer-use models: + +```typescript +const agent = stagehand.agent({ + mode: "cua", // Enable Computer Use Agent mode + model: "anthropic/claude-sonnet-4-20250514", + // or "google/gemini-2.5-computer-use-preview-10-2025" + systemPrompt: `You are a helpful assistant that can use a web browser. + Do not ask follow up questions, the user will trust your judgement.`, +}); + +await agent.execute({ + instruction: "Apply for a library card at the San Francisco Public Library", + maxSteps: 30, +}); +``` + +### Agent with Custom Model Configuration + +```typescript +const agent = stagehand.agent({ + model: { + modelName: "google/gemini-2.5-computer-use-preview-10-2025", + apiKey: process.env.GEMINI_API_KEY, + }, + systemPrompt: `You are a helpful assistant.`, +}); +``` + +### Agent with Integrations (MCP/External Tools) + +```typescript +const agent = stagehand.agent({ + integrations: [`https://mcp.exa.ai/mcp?exaApiKey=${process.env.EXA_API_KEY}`], + systemPrompt: `You have access to the Exa search tool.`, +}); +``` + +### Agent Hybrid Mode + +Hybrid mode uses both DOM-based and coordinate-based tools (act, click, type, dragAndDrop) for visual interactions. This requires `experimental: true` and models that support reliable coordinate-based actions. + +**Recommended models for hybrid mode:** + +- Any Anthropic model (e.g. `anthropic/claude-sonnet-4-20250514`, `anthropic/claude-haiku-4-5-20251001`) +- OpenAI: `openai/gpt-5.4`, `openai/gpt-5.4-mini` +- Google: `google/gemini-3.1-flash-lite-preview`, `google/gemini-3-flash-preview`, `google/gemini-3.1-pro-preview` + +```typescript +const stagehand = new Stagehand({ + env: "LOCAL", + experimental: true, // Required for hybrid mode +}); +await stagehand.init(); + +const agent = stagehand.agent({ + mode: "hybrid", + model: "google/gemini-3-flash-preview", +}); + +await agent.execute({ + instruction: "Click the submit button and fill the form", + maxSteps: 20, + highlightCursor: true, // Enabled by default in hybrid mode +}); +``` + +**Agent modes:** + +- `"dom"`: Uses DOM-based tools (act, fillForm) - works with any model +- `"hybrid"`: Uses both DOM-based and coordinate-based tools (act, click, type, dragAndDrop) - auto-selected for supported models (Claude, GPT-5.4, Gemini 3) +- When no mode is specified, hybrid is auto-selected for supported models; otherwise falls back to dom +- `"cua"`: Uses Computer Use Agent providers + +## Advanced Features + +### DeepLocator (XPath Targeting) + +Target specific elements across shadow DOM and iframes: + +```typescript +await page + .deepLocator("/html/body/div[2]/div[3]/iframe/html/body/p") + .highlight({ + durationMs: 5000, + contentColor: { r: 255, g: 0, b: 0 }, + }); +``` + +### Multi-Page Workflows + +```typescript +const page1 = stagehand.context.pages()[0]; +await page1.goto("https://example.com"); + +const page2 = await stagehand.context.newPage(); +await page2.goto("https://example2.com"); + +// Act/extract/observe operate on the current active page by default +// Pass { page } option to target a specific page +await stagehand.act("click button", { page: page1 }); +await stagehand.extract("get title", { page: page2 }); +``` diff --git a/docs/logging.md b/docs/logging.md deleted file mode 100644 index fa73482c1..000000000 --- a/docs/logging.md +++ /dev/null @@ -1,184 +0,0 @@ -# Stagehand Logging System - -The Stagehand logging system uses [Pino](https://getpino.io/) to provide structured, efficient, and configurable logging. - -## Log Levels - -Stagehand uses three primary log levels: - -| Level | Name | Description | -| ----- | ----- | --------------------------------------- | -| 0 | error | Critical errors and important warnings | -| 1 | info | Standard information messages (default) | -| 2 | debug | Detailed information for debugging | - -The verbosity of logging is controlled by the `verbose` option when creating a Stagehand instance: - -```typescript -const stagehand = new Stagehand({ - verbose: 2, // Show all logs up to debug level - // other options... -}); -``` - -## Using the Logger - -The logging system is automatically initialized with your Stagehand instance. You can access it directly via: - -```typescript -// Log an error -stagehand.log({ - message: "An error occurred", - level: 0, - category: "error", -}); - -// Log info (level 1 is default) -stagehand.log({ - message: "Operation completed", - category: "operation", -}); - -// Log debug information -stagehand.log({ - message: "Debug details", - level: 2, - category: "debug", - auxiliary: { - details: { - value: JSON.stringify({ key: "value" }), - type: "object", - }, - }, -}); -``` - -## Inference Logging - -For detailed logging of inference operations (act, extract, observe), Stagehand provides specialized logging: - -```typescript -// Enable inference logging to file -const stagehand = new Stagehand({ - logInferenceToFile: true, - // other options... -}); -``` - -When enabled, inference logs are written to the `inference_summary` directory in your project. - -## Pretty Printing - -By default, logs in development are formatted with colors and readable timestamps using Pino's pretty formatting. For production environments or when sending logs to external systems, you can disable pretty printing. - -## Customizing Logging - -### Using Your Own Logger - -You can provide your own custom logger when creating a Stagehand instance: - -```typescript -const stagehand = new Stagehand({ - logger: (logLine) => { - // Your custom logging logic here - console.log(`[${logLine.category}] ${logLine.message}`); - }, - // other options... -}); -``` - -When you provide a custom logger, Stagehand will automatically disable its internal Pino logger to prevent duplicate logging. Your logger will receive all log events directly. - -### Configuring Pino - -If you want to use Pino but with custom configuration: - -```typescript -import { StagehandLogger } from "@browserbasehq/stagehand/lib/logger"; - -// Create a custom configured logger -const customLogger = new StagehandLogger({ - pretty: true, - level: "debug", - // Other Pino options... -}); - -// Pass it to Stagehand -const stagehand = new Stagehand({ - logger: (logLine) => customLogger.log(logLine), - // other options... -}); -``` - -## Advanced Usage - -### Creating a New StagehandLogger Instance - -You can create a standalone logger for use in your application: - -```typescript -import { StagehandLogger } from "@browserbasehq/stagehand/lib/logger"; - -const logger = new StagehandLogger({ - pretty: true, - level: "debug", -}); - -logger.info("Information message"); -logger.debug("Debug message", { details: "some data" }); -logger.error("Error message", { error: "details" }); -``` - -### Configuring Log Output - -You can direct logs to a file or other destination: - -```typescript -import fs from "fs"; -import { StagehandLogger } from "@browserbasehq/stagehand/lib/logger"; - -const fileStream = fs.createWriteStream("./logs/application.log", { - flags: "a", -}); - -const logger = new StagehandLogger({ - destination: fileStream, -}); -``` - -### Disabling Pino Explicitly - -If you want to handle all logging yourself without using Pino: - -```typescript -import { StagehandLogger } from "@browserbasehq/stagehand/lib/logger"; - -const logger = new StagehandLogger( - { - usePino: false, - }, - (logLine) => { - // Your custom logging logic - console.log(`[${logLine.level}] ${logLine.message}`); - }, -); -``` - -## Troubleshooting - -If you're not seeing logs: - -1. Check your `verbose` setting - it may be too low for the log levels you're trying to see -2. Verify that your log messages have the correct level set -3. If using a custom logger, ensure it's correctly handling the log messages - -If you're seeing duplicate logs: - -1. Make sure you're not creating multiple instances of StagehandLogger that log to the same output -2. If providing a custom logger to Stagehand, it will automatically disable the internal Pino logger - -If logs are not being written to files: - -1. Ensure you have write permissions to the target directory -2. Check that the `logInferenceToFile` option is enabled -3. Verify that the destination path exists or can be created diff --git a/docs/release.md b/docs/release.md deleted file mode 100644 index 2a0402a1a..000000000 --- a/docs/release.md +++ /dev/null @@ -1,117 +0,0 @@ -# Releasing - -We use [Changesets](https://github.com/changesets/changesets) to version and release our packages. - -When we merge to main, the release workflow will: - -1. Create a release pull request with: - - A version bump for the package calculated by the changesets. - - A changelog entry summarizing the changes in the release. -1. Create an `alpha` version of the package with whatever is merged to main, and you can install it with `npm install @browserbasehq/stagehand@alpha`. This is useful for testing the release before it's published to the `latest` tag. - -When the pull request is merged, the release workflow will publish the package to npm with the version calculated by the changesets. - -For more information on how changesets work, see the [changesets docs](https://github.com/changesets/changesets) and our [release.yml file](/.github/workflows/release.yml). - -# Manually Releasing - -> [!WARNING] -> You should not need to manually release unless absolutely necessary. Our automated release workflow handles this for you when changes are merged to main. - -When you're ready to cut a release, start by versioning the packages: - -``` -npx changeset version -``` - -This will consume the changesets in [`.changeset`](../.changeset) and update the [changelog](../CHANGELOG.md) and [`package.json`](../package.json): - -``` -% git status --short - M CHANGELOG.md - M package.json -``` - -Based on the versions implications declared by the changesets, the package version will be updated to the next patch, minor, or major: - -```diff - "name": "@browserbasehq/stagehand", -- "version": "1.3.0", -+ "version": "1.3.1", -``` - -Since we updated the `package.json`, we should also update the lockfile ([`package-lock.json`](../package-lock.json)) for tidiness: - -``` -npm install -``` - -Now the lockfile should be updated: - -``` -% git status --short - M CHANGELOG.md - M package-lock.json - M package.json -``` - -The diff will look something like this: - -```diff - { - "name": "@browserbasehq/stagehand", -- "version": "1.3.0", -+ "version": "1.3.1", - "lockfileVersion": 3, - "requires": true, - "packages": { - "": { - "name": "@browserbasehq/stagehand", -- "version": "1.3.0", -+ "version": "1.3.1", -``` - -At this point we're ready to commit our changes. -It's probably a good idea to have some consistency around the name of this commit message: - -``` -git commit -am 'Version Packages' -``` - -Ok, now it's time to publish the release. -Before we do, we have to build the artifacts that comprise the tarball. -Let's clean our working directory first so that we don't accidentally include anything in the tarball that shouldn't be there: - -``` -% git clean -fxd -e .env -Removing dist/ -Removing lib/dom/build/ -Removing node_modules/ -``` - -Let's reinstall dependencies and build the artifacts: - -``` -npm install && npm run build -``` - -Now we're ready to publish to NPM. You have to be logged in via the `npm` CLI and have to be part of the `@browserbasehq` org: - -``` -npx changeset publish -``` - -Congratulations! You just published a new version of `@browserbasehq/stagehand`. 🤘 - -In the process of publishing, Changesets created an [annotated git tag](https://git-scm.com/book/en/v2/Git-Basics-Tagging): - -``` -🦋 Creating git tag... -🦋 New tag: v1.3.1 -``` - -Let's push the commit and tag to GitHub for posterity: - -``` -git push --follow-tags -``` diff --git a/eslint.config.mjs b/eslint.config.mjs index c0d192cde..0342ec4b8 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -1,12 +1,86 @@ import globals from "globals"; import pluginJs from "@eslint/js"; import tseslint from "typescript-eslint"; +import security from "eslint-plugin-security"; /** @type {import('eslint').Linter.Config[]} */ export default [ { files: ["**/*.{js,mjs,cjs,ts}"] }, { languageOptions: { globals: globals.browser } }, - { ignores: ["**/dist/**", "lib/dom/build/**"] }, + { + files: ["packages/core/scripts/**/*.{js,cjs,mjs}"], + languageOptions: { globals: globals.node }, + }, + { + files: [ + "packages/server-v3/scripts/**/*.{js,cjs,mjs,ts}", + "packages/server-v4/scripts/**/*.{js,cjs,mjs,ts}", + ], + languageOptions: { globals: globals.node }, + }, + { + files: ["packages/cli/**/*.{js,cjs,mjs,ts}"], + languageOptions: { globals: globals.node }, + }, + { + ignores: [ + "**/dist/**", + "**/node_modules/**", + "packages/core/lib/dom/build/**", + "packages/core/lib/v3/dom/build/**", + "packages/core/lib/v4/dom/build/**", + "packages/core/scripts/prepare.js", + "**/*.config.js", + "**/*.config.mjs", + ".browserbase/**", + "**/.browserbase/**", + "**/*.json", + "stainless.yml", + "packages/server-v3/openapi.v3.yaml", + "packages/server-v4/openapi.v4.yaml", + ], + }, pluginJs.configs.recommended, ...tseslint.configs.recommended, + { + plugins: { + security, + }, + rules: { + "no-eval": "error", + "no-implied-eval": "error", + "no-new-func": "error", + "security/detect-eval-with-expression": "error", + "preserve-caught-error": "error", + "no-restricted-syntax": [ + "error", + { + selector: "CallExpression[callee.name='Function']", + message: "Dynamic function construction is prohibited.", + }, + { + selector: "NewExpression[callee.name='Function']", + message: "Dynamic function construction is prohibited.", + }, + { + selector: + "CallExpression[callee.object.name='window'][callee.property.name='Function']", + message: + "Dynamic function construction via window.Function is prohibited.", + }, + { + selector: + "CallExpression[callee.object.name='globalThis'][callee.property.name='Function']", + message: + "Dynamic function construction via globalThis.Function is prohibited.", + }, + ], + }, + }, + { + files: ["packages/cli/**/*.{js,cjs,mjs,ts}"], + rules: { + "no-empty": ["error", { allowEmptyCatch: true }], + }, + }, ]; diff --git a/evals/args.ts b/evals/args.ts deleted file mode 100644 index 4c2f748fc..000000000 --- a/evals/args.ts +++ /dev/null @@ -1,110 +0,0 @@ -import process from "process"; -import { EvalCategorySchema } from "@/types/evals"; - -const rawArgs = process.argv.slice(2); - -const parsedArgs: { - env?: string; - trials?: number; - concurrency?: number; - extractMethod?: string; - provider?: string; - leftover: string[]; -} = { - leftover: [], -}; - -for (const arg of rawArgs) { - if (arg.startsWith("env=")) { - parsedArgs.env = arg.split("=")[1]?.toLowerCase(); - } else if (arg.startsWith("trials=")) { - const val = parseInt(arg.split("=")[1], 10); - if (!isNaN(val)) { - parsedArgs.trials = val; - } - } else if (arg.startsWith("concurrency=")) { - const val = parseInt(arg.split("=")[1], 10); - if (!isNaN(val)) { - parsedArgs.concurrency = val; - } - } else if (arg.startsWith("--extract-method=")) { - parsedArgs.extractMethod = arg.split("=")[1]; - } else if (arg.startsWith("provider=")) { - parsedArgs.provider = arg.split("=")[1]?.toLowerCase(); - } else { - parsedArgs.leftover.push(arg); - } -} - -/** Apply environment defaults or overrides */ -if (parsedArgs.env === "browserbase") { - process.env.EVAL_ENV = "BROWSERBASE"; -} else if (parsedArgs.env === "local") { - process.env.EVAL_ENV = "LOCAL"; -} - -if (parsedArgs.trials !== undefined) { - process.env.EVAL_TRIAL_COUNT = String(parsedArgs.trials); -} -if (parsedArgs.concurrency !== undefined) { - process.env.EVAL_MAX_CONCURRENCY = String(parsedArgs.concurrency); -} - -const extractMethod = parsedArgs.extractMethod || "domExtract"; -process.env.EXTRACT_METHOD = extractMethod; - -const useTextExtract = extractMethod === "textExtract"; -const useAccessibilityTree = extractMethod === "accessibilityTree"; - -const DEFAULT_EVAL_CATEGORIES = process.env.EVAL_CATEGORIES - ? process.env.EVAL_CATEGORIES.split(",") - : [ - "observe", - "act", - "combination", - "extract", - "experimental", - "text_extract", - "targeted_extract", - "regression_llm_providers", - "regression", - "llm_clients", - ]; - -// Finally, interpret leftover arguments to see if user typed "category X" or a single eval name -let filterByCategory: string | null = null; -let filterByEvalName: string | null = null; - -if (parsedArgs.leftover.length > 0) { - if (parsedArgs.leftover[0].toLowerCase() === "category") { - filterByCategory = parsedArgs.leftover[1]; - if (!filterByCategory) { - console.error("Error: Category name not specified."); - process.exit(1); - } - try { - EvalCategorySchema.parse(filterByCategory); - } catch { - console.error( - `Error: Invalid category "${filterByCategory}". Valid categories are: ${DEFAULT_EVAL_CATEGORIES.join(", ")}`, - ); - process.exit(1); - } - } else { - // If leftover[0] is not "category", interpret it as a task/eval name - filterByEvalName = parsedArgs.leftover[0]; - } -} - -if (parsedArgs.provider !== undefined) { - process.env.EVAL_PROVIDER = parsedArgs.provider; -} - -export { - filterByCategory, - filterByEvalName, - useTextExtract, - useAccessibilityTree, - DEFAULT_EVAL_CATEGORIES, - parsedArgs, -}; diff --git a/evals/deterministic/auxiliary/logo.png b/evals/deterministic/auxiliary/logo.png deleted file mode 100644 index 7ae5d4850..000000000 Binary files a/evals/deterministic/auxiliary/logo.png and /dev/null differ diff --git a/evals/deterministic/bb.playwright.config.ts b/evals/deterministic/bb.playwright.config.ts deleted file mode 100644 index cea358f55..000000000 --- a/evals/deterministic/bb.playwright.config.ts +++ /dev/null @@ -1,31 +0,0 @@ -import { defineConfig, devices } from "@playwright/test"; - -/** - * See https://playwright.dev/docs/test-configuration. - */ -export default defineConfig({ - testDir: "./tests/browserbase", - - /* Fail the build on CI if you accidentally left test.only in the source code. */ - /* Run tests in files in parallel */ - fullyParallel: true, - /* Reporter to use. See https://playwright.dev/docs/test-reporters */ - // reporter: "html", - reporter: "line", - /* Retry on CI only */ - retries: 2, - - /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ - use: { - /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */ - trace: "on-first-retry", - }, - - /* Configure projects for major browsers */ - projects: [ - { - name: "chromium", - use: { ...devices["Desktop Chrome"] }, - }, - ], -}); diff --git a/evals/deterministic/e2e.playwright.config.ts b/evals/deterministic/e2e.playwright.config.ts deleted file mode 100644 index 4ed3c2594..000000000 --- a/evals/deterministic/e2e.playwright.config.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { defineConfig, devices } from "@playwright/test"; - -/** - * See https://playwright.dev/docs/test-configuration. - */ -export default defineConfig({ - // Look in "tests" for test files... - testDir: "./tests", - // ...but ignore anything in "tests/browserbase & "tests/local" - testIgnore: ["**/browserbase/**", "**/local/**"], - - /* Fail the build on CI if you accidentally left test.only in the source code. */ - /* Run tests in files in parallel */ - fullyParallel: true, - /* Reporter to use. See https://playwright.dev/docs/test-reporters */ - // reporter: "html", - reporter: "line", - retries: 2, - - /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ - use: { - /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */ - trace: "on-first-retry", - }, - - /* Configure projects for major browsers */ - projects: [ - { - name: "chromium", - use: { ...devices["Desktop Chrome"] }, - }, - ], -}); diff --git a/evals/deterministic/local.playwright.config.ts b/evals/deterministic/local.playwright.config.ts deleted file mode 100644 index 3c309ff70..000000000 --- a/evals/deterministic/local.playwright.config.ts +++ /dev/null @@ -1,37 +0,0 @@ -import { defineConfig, devices } from "@playwright/test"; - -/** - * See https://playwright.dev/docs/test-configuration. - */ -export default defineConfig({ - testDir: "./tests/local", - - /* Maximum time one test can run for. */ - timeout: 30 * 1000, - - /* Fail the build on CI if you accidentally left test.only in the source code. */ - forbidOnly: !!process.env.CI, - - /* Run tests in files in parallel */ - fullyParallel: false, - - /* Reporter to use */ - reporter: "line", - - /* Retry on CI only */ - retries: process.env.CI ? 2 : 0, - - /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ - use: { - /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */ - trace: "on-first-retry", - }, - - /* Configure projects for major browsers */ - projects: [ - { - name: "chromium", - use: { ...devices["Desktop Chrome"] }, - }, - ], -}); diff --git a/evals/deterministic/stagehand.config.ts b/evals/deterministic/stagehand.config.ts deleted file mode 100644 index a7bcbb644..000000000 --- a/evals/deterministic/stagehand.config.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { default as DefaultStagehandConfig } from "@/stagehand.config"; -import type { ConstructorParams } from "@/dist"; -import dotenv from "dotenv"; -dotenv.config({ path: "../../.env" }); - -const StagehandConfig: ConstructorParams = { - ...DefaultStagehandConfig, - env: "LOCAL" /* Environment to run Stagehand in */, - verbose: 1 /* Logging verbosity level (0=quiet, 1=normal, 2=verbose) */, - browserbaseSessionCreateParams: { - projectId: process.env.BROWSERBASE_PROJECT_ID, - }, - enableCaching: false /* Enable caching functionality */, - localBrowserLaunchOptions: { - headless: true /* Run browser in headless mode */, - }, -}; -export default StagehandConfig; diff --git a/evals/deterministic/tests/BrowserContext/addInitScript.test.ts b/evals/deterministic/tests/BrowserContext/addInitScript.test.ts deleted file mode 100644 index cf78933c2..000000000 --- a/evals/deterministic/tests/BrowserContext/addInitScript.test.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { test, expect } from "@playwright/test"; -import { Stagehand } from "@/dist"; -import StagehandConfig from "@/evals/deterministic/stagehand.config"; - -test.describe("StagehandContext - addInitScript", () => { - test("should inject a script on the context before pages load", async () => { - const stagehand = new Stagehand(StagehandConfig); - await stagehand.init(); - - const context = stagehand.context; - - await context.addInitScript(() => { - const w = window as typeof window & { - __testContextScriptVar?: string; - }; - w.__testContextScriptVar = "Hello from context.initScript!"; - }); - - const pageA = await context.newPage(); - await pageA.goto("https://example.com"); - - const resultA = await pageA.evaluate(() => { - const w = window as typeof window & { - __testContextScriptVar?: string; - }; - return w.__testContextScriptVar; - }); - expect(resultA).toBe("Hello from context.initScript!"); - - const pageB = await context.newPage(); - await pageB.goto("https://docs.browserbase.com"); - - const resultB = await pageB.evaluate(() => { - const w = window as typeof window & { - __testContextScriptVar?: string; - }; - return w.__testContextScriptVar; - }); - expect(resultB).toBe("Hello from context.initScript!"); - - await stagehand.close(); - }); -}); diff --git a/evals/deterministic/tests/BrowserContext/cookies.test.ts b/evals/deterministic/tests/BrowserContext/cookies.test.ts deleted file mode 100644 index b42c04507..000000000 --- a/evals/deterministic/tests/BrowserContext/cookies.test.ts +++ /dev/null @@ -1,69 +0,0 @@ -import { test, expect } from "@playwright/test"; -import { Stagehand } from "@/dist"; -import StagehandConfig from "@/evals/deterministic/stagehand.config"; - -test.describe("StagehandContext - Cookies", () => { - let stagehand: Stagehand; - - test.beforeEach(async () => { - stagehand = new Stagehand(StagehandConfig); - await stagehand.init(); - }); - - test.afterEach(async () => { - await stagehand.close(); - }); - - test("should add cookies and retrieve them", async () => { - const context = stagehand.context; // This is the wrapped BrowserContext - const url = "https://example.com"; - - await context.addCookies([ - { - name: "myCookie", - value: "myValue", - domain: "example.com", - path: "/", - expires: Math.floor(Date.now() / 1000) + 3600, - httpOnly: false, - secure: false, - sameSite: "Lax", - }, - ]); - - const cookies = await context.cookies(url); - expect(cookies.length).toBeGreaterThan(0); - - const myCookie = cookies.find((c) => c.name === "myCookie"); - expect(myCookie).toBeDefined(); - expect(myCookie?.value).toBe("myValue"); - }); - - test("should clear all cookies", async () => { - const context = stagehand.context; - const url = "https://example.com"; - - await context.addCookies([ - { - name: "myOtherCookie", - value: "anotherValue", - domain: "example.com", - path: "/", - expires: Math.floor(Date.now() / 1000) + 3600, - httpOnly: false, - secure: false, - sameSite: "Lax", - }, - ]); - - const cookiesBefore = await context.cookies(url); - const found = cookiesBefore.some((c) => c.name === "myOtherCookie"); - expect(found).toBe(true); - - await context.clearCookies(); - - const cookiesAfter = await context.cookies(url); - const stillFound = cookiesAfter.some((c) => c.name === "myOtherCookie"); - expect(stillFound).toBe(false); - }); -}); diff --git a/evals/deterministic/tests/BrowserContext/multiPage.test.ts b/evals/deterministic/tests/BrowserContext/multiPage.test.ts deleted file mode 100644 index b0b38cdf0..000000000 --- a/evals/deterministic/tests/BrowserContext/multiPage.test.ts +++ /dev/null @@ -1,209 +0,0 @@ -import { test, expect } from "@playwright/test"; -import { Stagehand } from "@/dist"; -import StagehandConfig from "@/evals/deterministic/stagehand.config"; -import { Page } from "@/dist"; - -import http from "http"; -import express from "express"; -import { Server as WebSocketServer } from "ws"; - -test.describe("StagehandContext - Multi-page Support", () => { - let stagehand: Stagehand; - let server: http.Server; - let wss: WebSocketServer; - let serverPort: number; - - test.beforeAll(async () => { - // Set up a local Express server - const app = express(); - - // Serve test pages - app.get("/page1", (_req, res) => { - res.set("Content-Type", "text/html"); - res.end(` - - Page 1 - -

    Page 1 Content

    - - - - `); - }); - - app.get("/page2", (_req, res) => { - res.set("Content-Type", "text/html"); - res.end(` - - Page 2 - -

    Page 2 Content

    - - - `); - }); - - // Create the server on a random free port - server = http.createServer(app); - await new Promise((resolve) => { - server.listen(0, () => resolve()); - }); - const address = server.address(); - if (typeof address === "object" && address !== null) { - serverPort = address.port; - } else { - throw new Error("Failed to get server port"); - } - - // Set up WebSocket for future tests - wss = new WebSocketServer({ server, path: "/socket" }); - wss.on("connection", (ws) => { - console.log("WebSocket client connected"); - ws.send("Hello from server WebSocket"); - }); - }); - - test.beforeEach(async () => { - stagehand = new Stagehand(StagehandConfig); - await stagehand.init(); - }); - - test.afterEach(async () => { - await stagehand.close(); - }); - - test.afterAll(async () => { - wss?.close(); - server?.close(); - }); - - /** - * Test enhanced page capabilities - */ - test("should provide enhanced capabilities for new pages", async () => { - const context = stagehand.context; - const newPage = await context.newPage(); - - // Verify enhanced methods - expect(typeof newPage.act).toBe("function"); - expect(typeof newPage.extract).toBe("function"); - expect(typeof newPage.observe).toBe("function"); - - // Verify basic Playwright functionality - expect(typeof newPage.goto).toBe("function"); - expect(typeof newPage.click).toBe("function"); - - // Test navigation maintains capabilities - await newPage.goto(`http://localhost:${serverPort}/page1`); - expect(typeof newPage.act).toBe("function"); - expect(await newPage.title()).toBe("Page 1"); - }); - - /** - * Test context.pages() functionality - */ - test("should return array of enhanced pages via context.pages()", async () => { - const context = stagehand.context; - - // Create multiple pages - const page1 = await context.newPage(); - const page2 = await context.newPage(); - - await page1.goto(`http://localhost:${serverPort}/page1`); - await page2.goto(`http://localhost:${serverPort}/page2`); - - const pages = context.pages(); - expect(pages).toContain(page1); - expect(pages).toContain(page2); - - // Verify all pages have enhanced capabilities - for (const page of pages) { - expect(typeof page.act).toBe("function"); - expect(typeof page.extract).toBe("function"); - expect(typeof page.observe).toBe("function"); - } - }); - - /** - * Test popup handling - */ - test("should handle popups with enhanced capabilities", async () => { - const mainPage = stagehand.page; - let popupPage: Page | null = null; - - mainPage.on("popup", (page: Page) => { - popupPage = page; - }); - - await mainPage.goto(`http://localhost:${serverPort}/page1`); - await mainPage.click("#popupBtn"); - - // Verify popup has enhanced capabilities - expect(popupPage).not.toBeNull(); - expect(typeof popupPage.act).toBe("function"); - expect(typeof popupPage.extract).toBe("function"); - expect(typeof popupPage.observe).toBe("function"); - - if (popupPage) { - await popupPage.waitForLoadState(); - expect(await popupPage.title()).toBe("Page 2"); - } - }); - - /** - * Test page tracking and cleanup - */ - test("should properly track and cleanup pages", async () => { - const context = stagehand.context; - const initialPages = context.pages().length; - - const newPage = await context.newPage(); - await newPage.goto(`http://localhost:${serverPort}/page1`); - - expect(context.pages().length).toBe(initialPages + 1); - await newPage.close(); - expect(context.pages().length).toBe(initialPages); - }); - - /** - * Test enhanced methods across pages - */ - test("should support enhanced methods across all pages", async () => { - const page1 = await stagehand.context.newPage(); - const page2 = await stagehand.context.newPage(); - - await page1.goto(`http://localhost:${serverPort}/page1`); - await page2.goto(`http://localhost:${serverPort}/page2`); - - // Verify both pages have enhanced capabilities - expect(typeof page1.act).toBe("function"); - expect(typeof page1.extract).toBe("function"); - expect(typeof page1.observe).toBe("function"); - - expect(typeof page2.act).toBe("function"); - expect(typeof page2.extract).toBe("function"); - expect(typeof page2.observe).toBe("function"); - }); - - /** - * Test active page tracking - */ - test("should update stagehand.page when creating new pages", async () => { - const initialPage = stagehand.page; - - // Create a new page and verify it becomes active - const newPage = await stagehand.context.newPage(); - expect(stagehand.page).toBe(newPage); - expect(stagehand.page).not.toBe(initialPage); - - // Navigate and verify it's still the active page - await newPage.goto(`http://localhost:${serverPort}/page1`); - expect(stagehand.page).toBe(newPage); - expect(await stagehand.page.title()).toBe("Page 1"); - - // Create another page and verify it becomes active - const anotherPage = await stagehand.context.newPage(); - expect(stagehand.page).toBe(anotherPage); - expect(stagehand.page).not.toBe(newPage); - }); -}); diff --git a/evals/deterministic/tests/BrowserContext/page.test.ts b/evals/deterministic/tests/BrowserContext/page.test.ts deleted file mode 100644 index f6c7aca58..000000000 --- a/evals/deterministic/tests/BrowserContext/page.test.ts +++ /dev/null @@ -1,100 +0,0 @@ -import { test, expect } from "@playwright/test"; -import { Stagehand } from "@/dist"; -import StagehandConfig from "@/evals/deterministic/stagehand.config"; - -import http from "http"; -import express from "express"; -import { Server as WebSocketServer } from "ws"; - -test.describe("StagehandContext - pages and newPage", () => { - let stagehand: Stagehand; - let server: http.Server; - let wss: WebSocketServer; - let serverPort: number; - - test.beforeAll(async () => { - // 1. Spin up a local Express server - const app = express(); - - // Serve a single page at "/" - app.get("/", (_req, res) => { - res.set("Content-Type", "text/html"); - res.end(` - - - Test Page - - -

    Hello from local server

    - - - - `); - }); - - // Create the server on a random free port - server = http.createServer(app); - await new Promise((resolve) => { - server.listen(0, () => resolve()); - }); - const address = server.address(); - if (typeof address === "object" && address !== null) { - serverPort = address.port; - } else { - throw new Error("Failed to get server port"); - } - - // Optionally set up a WebSocket for future tests - wss = new WebSocketServer({ server, path: "/socket" }); - wss.on("connection", (ws) => { - console.log("WebSocket client connected"); - ws.send("Hello from server WebSocket"); - }); - }); - - test.beforeEach(async () => { - // 2. Create & init Stagehand for each test - stagehand = new Stagehand(StagehandConfig); - await stagehand.init(); - }); - - test.afterEach(async () => { - await stagehand.close(); - }); - - test.afterAll(async () => { - // Shut down local server - wss?.close(); - server?.close(); - }); - - /** - * Test context.newPage() and context.pages() - */ - test("should create multiple pages and list them via context.pages()", async () => { - const context = stagehand.context; - - // Create multiple pages - const page1 = await context.newPage(); - const page2 = await context.newPage(); - - // Confirm context.pages() sees them - const allPages = context.pages(); - - // We expect at least these 2 pages. If a default blank page existed, total might be more. - // The key is that page1 & page2 are in the array: - expect(allPages).toContain(page1); - expect(allPages).toContain(page2); - - // Navigate page1 to the local server - await page1.goto(`http://localhost:${serverPort}`); - expect(await page1.title()).toBe("Test Page"); - }); -}); diff --git a/evals/deterministic/tests/BrowserContext/routing.test.ts b/evals/deterministic/tests/BrowserContext/routing.test.ts deleted file mode 100644 index 1dd53664d..000000000 --- a/evals/deterministic/tests/BrowserContext/routing.test.ts +++ /dev/null @@ -1,236 +0,0 @@ -import { test, expect } from "@playwright/test"; -import { Stagehand } from "@/dist"; -import StagehandConfig from "@/evals/deterministic/stagehand.config"; - -import http from "http"; -import express from "express"; -import { Server as WebSocketServer } from "ws"; -import fs from "fs"; -import path from "path"; - -const HAR_CONTENT = `{ - "log": { - "version": "1.2", - "creator": { "name": "PlaywrightTest", "version": "1.0" }, - "entries": [ - { - "startedDateTime": "2023-01-01T00:00:00.000Z", - "time": 5, - "request": { - "method": "GET", - "url": "http://localhost/har-example.json", - "httpVersion": "HTTP/1.1", - "cookies": [], - "headers": [], - "queryString": [], - "headersSize": -1, - "bodySize": 0 - }, - "response": { - "status": 200, - "statusText": "OK", - "httpVersion": "HTTP/1.1", - "cookies": [], - "headers": [{"name":"Content-Type","value":"application/json"}], - "content": { - "size": 27, - "mimeType": "application/json", - "text": "{\\"harKey\\":\\"harValue\\"}" - }, - "redirectURL": "", - "headersSize": -1, - "bodySize": 0 - }, - "cache": {}, - "timings": { "send": 0, "wait": 5, "receive": 0 } - } - ] - } -}`; - -test.describe("StagehandContext - Routing APIs with dynamic setup", () => { - let stagehand: Stagehand; - let server: http.Server; - let wss: WebSocketServer; - let serverPort: number; - - test.beforeAll(async () => { - const app = express(); - - app.get("/example.json", (_req, res) => { - res.json({ original: "server-data" }); - }); - - app.get("/har-example.json", (_req, res) => { - res.json({ - fromServer: - "This should be replaced by HAR if routeFromHar is in effect", - }); - }); - - server = http.createServer(app); - await new Promise((resolve) => { - server.listen(0, () => resolve()); - }); - const address = server.address(); - if (typeof address === "object" && address !== null) { - serverPort = address.port; - } else { - throw new Error("Failed to get server port"); - } - - // Set up a WebSocket endpoint at "/socket" - wss = new WebSocketServer({ server, path: "/socket" }); - wss.on("connection", (ws) => { - console.log("WebSocket client connected"); - ws.send("Hello from server WebSocket"); - - // Echo messages back - ws.on("message", (message) => { - console.log("Server received WS message:", message); - ws.send(`Server echo: ${message}`); - }); - }); - }); - - test.beforeEach(async () => { - stagehand = new Stagehand(StagehandConfig); - await stagehand.init(); - }); - - test.afterEach(async () => { - await stagehand.close(); - }); - - test.afterAll(async () => { - wss?.close(); - server?.close(); - }); - - test("should intercept requests, mock the response, handle websockets, and unroute them", async () => { - const context = stagehand.context; - const baseURL = `http://localhost:${serverPort}`; - - // 1. route: intercept "/example.json" and fulfill with a mock response - await context.route("**/example.json", async (route) => { - console.log("[route] Intercepting:", route.request().url()); - - // Mock the response entirely: - await route.fulfill({ - status: 200, - contentType: "application/json", - body: JSON.stringify({ mockedData: 1234 }), - }); - }); - - // 2. routeWebSocket: intercept "/socket" - await context.routeWebSocket("**/socket", async (pageSideRoute) => { - console.log("Intercepting WebSocket at:", pageSideRoute.url()); - - // Connect to the real server - const serverSideRoute = pageSideRoute.connectToServer(); - - // Page -> Server - pageSideRoute.onMessage((msg) => { - console.log("Page -> Server message:", msg); - // Forward to server side - serverSideRoute.send(msg); - }); - - // Server -> Page - serverSideRoute.onMessage((msg) => { - console.log("Server -> Page message:", msg); - pageSideRoute.send(msg); - }); - }); - - // 3. Open a page and fetch /example.json - const page = await context.newPage(); - await page.goto(baseURL); - - const fetchResult = await page.evaluate(async () => { - const res = await fetch("/example.json"); - return res.json(); - }); - // We should get the mocked data from our route, not the real 'server-data' - expect(fetchResult.mockedData).toBe(1234); - - // 4. Test the WebSocket - // We'll store messages from the server in an array so we can assert them - const wsMessages: string[] = []; - page.on("console", (msg) => { - // We'll parse out the console logs we used for WebSocket - if (msg.type() === "log") { - wsMessages.push(msg.text()); - } - }); - - // Create a WS from the page - await page.evaluate((port) => { - const ws = new WebSocket(`ws://localhost:${port}/socket`); - ws.onmessage = (evt) => { - console.log(`WS message from server: ${evt.data}`); - }; - setTimeout(() => { - // send a message from the page side - ws.send("Hello from the client"); - }, 1000); - }, serverPort); - - // Wait a moment for messages - await page.waitForTimeout(3000); - - // We expect the server to have initially sent "Hello from server WebSocket" - // And also an echo of "Hello from the client" => "Server echo: Hello from the client" - const initialHello = wsMessages.find((m) => - m.includes("Hello from server WebSocket"), - ); - expect(initialHello).toBeTruthy(); - - const echoMessage = wsMessages.find((m) => - m.includes("Server echo: Hello from the client"), - ); - expect(echoMessage).toBeTruthy(); - - // 5. unroute the JSON route - await context.unroute("**/example.json"); - - // 6. confirm the WebSocket route is still active - // do a second fetch -> This time it won't be mocked - const fetchResult2 = await page.evaluate(async () => { - const res = await fetch("/example.json"); - return res.json(); - }); - // The real server returns { original: "server-data" } - expect(fetchResult2.original).toBe("server-data"); - - // 7. unrouteAll - await context.unrouteAll(); - }); - - test("should demonstrate routeFromHar usage", async () => { - const harPath = path.join(__dirname, "tmp-test.har"); - - const dynamicHar = HAR_CONTENT.replace( - "http://localhost/har-example.json", - `http://localhost:${serverPort}/har-example.json`, - ); - - fs.writeFileSync(harPath, dynamicHar, "utf-8"); - - const context = stagehand.context; - - await context.routeFromHAR(harPath, { update: false }); - - const page = await context.newPage(); - await page.goto(`http://localhost:${serverPort}/har-example.json`); - - const bodyText = await page.evaluate(() => document.body.innerText); - console.log("HAR-based body text:", bodyText); - expect(bodyText).toContain("harKey"); - expect(bodyText).toContain("harValue"); - - await context.unrouteAll(); - fs.unlinkSync(harPath); - }); -}); diff --git a/evals/deterministic/tests/Errors/apiKeyError.test.ts b/evals/deterministic/tests/Errors/apiKeyError.test.ts deleted file mode 100644 index 2cdc6656f..000000000 --- a/evals/deterministic/tests/Errors/apiKeyError.test.ts +++ /dev/null @@ -1,77 +0,0 @@ -import { test, expect } from "@playwright/test"; -import { Stagehand } from "@/dist"; -import StagehandConfig from "@/evals/deterministic/stagehand.config"; -import { z } from "zod"; - -test.describe("API key/LLMClient error", () => { - test("Should confirm that we get an error if we call extract without LLM API key or LLMClient", async () => { - const stagehand = new Stagehand(StagehandConfig); - await stagehand.init(); - await stagehand.page.goto("https://docs.browserbase.com/introduction"); - - let errorThrown: Error | null = null; - - try { - await stagehand.page.extract({ - instruction: - "From the introduction page, extract the explanation of what Browserbase is.", - schema: z.object({ - stars: z.string().describe("the explanation of what Browserbase is"), - }), - }); - } catch (error) { - errorThrown = error as Error; - } - - expect(errorThrown).toBeInstanceOf(Error); - expect(errorThrown?.message).toContain( - "No LLM API key or LLM Client configured", - ); - - await stagehand.close(); - }); - - test("Should confirm that we get an error if we call act without LLM API key or LLMClient", async () => { - const stagehand = new Stagehand(StagehandConfig); - await stagehand.init(); - await stagehand.page.goto("https://docs.browserbase.com/introduction"); - - let errorThrown: Error | null = null; - - try { - await stagehand.page.act({ - action: "Click on the 'Quickstart' section", - }); - } catch (error) { - errorThrown = error as Error; - } - - expect(errorThrown).toBeInstanceOf(Error); - expect(errorThrown?.message).toContain( - "No LLM API key or LLM Client configured", - ); - - await stagehand.close(); - }); - - test("Should confirm that we get an error if we call observe without LLM API key or LLMClient", async () => { - const stagehand = new Stagehand(StagehandConfig); - await stagehand.init(); - await stagehand.page.goto("https://docs.browserbase.com/introduction"); - - let errorThrown: Error | null = null; - - try { - await stagehand.page.observe(); - } catch (error) { - errorThrown = error as Error; - } - - expect(errorThrown).toBeInstanceOf(Error); - expect(errorThrown?.message).toContain( - "No LLM API key or LLM Client configured", - ); - - await stagehand.close(); - }); -}); diff --git a/evals/deterministic/tests/browserbase/contexts.test.ts b/evals/deterministic/tests/browserbase/contexts.test.ts deleted file mode 100644 index 0f8ecea9c..000000000 --- a/evals/deterministic/tests/browserbase/contexts.test.ts +++ /dev/null @@ -1,145 +0,0 @@ -import Browserbase from "@browserbasehq/sdk"; -import { expect, test } from "@playwright/test"; -import StagehandConfig from "@/evals/deterministic/stagehand.config"; -import { Stagehand } from "@/dist"; - -// Configuration -const CONTEXT_TEST_URL = "https://docs.browserbase.com"; -const BROWSERBASE_PROJECT_ID = process.env.BROWSERBASE_PROJECT_ID!; -const BROWSERBASE_API_KEY = process.env.BROWSERBASE_API_KEY!; - -const bb = new Browserbase({ - apiKey: BROWSERBASE_API_KEY, -}); - -// Helper functions -function addHour(date: Date): number { - const SECOND = 1000; - return new Date(date.getTime() + 60 * 60 * 1000).getTime() / SECOND; -} - -async function findCookie(stagehand: Stagehand, name: string) { - const defaultContext = stagehand.context; - const cookies = await defaultContext?.cookies(); - return cookies?.find((cookie) => cookie.name === name); -} - -async function createContext() { - console.log("Creating a new context..."); - const context = await bb.contexts.create({ - projectId: BROWSERBASE_PROJECT_ID, - }); - const contextId = context.id; - console.log(`Context created with ID: ${contextId}`); - return contextId; -} - -async function setRandomCookie(contextId: string, stagehand: Stagehand) { - console.log( - `Populating context ${contextId} during session ${stagehand.browserbaseSessionID}`, - ); - const page = stagehand.page; - - await page.goto(CONTEXT_TEST_URL, { waitUntil: "domcontentloaded" }); - - const now = new Date(); - const testCookieName = `bb_${now.getTime().toString()}`; - const testCookieValue = now.toISOString(); - - await stagehand.context.addCookies([ - { - domain: `.${new URL(CONTEXT_TEST_URL).hostname}`, - expires: addHour(now), - name: testCookieName, - path: "/", - value: testCookieValue, - }, - ]); - - expect(findCookie(stagehand, testCookieName)).toBeDefined(); - console.log(`Set test cookie: ${testCookieName}=${testCookieValue}`); - return { testCookieName, testCookieValue }; -} - -test.describe("Contexts", () => { - test("Persists and re-uses a context", async () => { - let contextId: string; - let testCookieName: string; - let testCookieValue: string; - let stagehand: Stagehand; - - await test.step("Create a context", async () => { - contextId = await createContext(); - }); - - await test.step("Instantiate Stagehand with the context to persist", async () => { - // We will be adding cookies to the context in this session, so we need mark persist=true - stagehand = new Stagehand({ - ...StagehandConfig, - browserbaseSessionCreateParams: { - projectId: BROWSERBASE_PROJECT_ID, - browserSettings: { - context: { - id: contextId, - persist: true, - }, - }, - }, - }); - await stagehand.init(); - }); - - await test.step("Set a random cookie on the page", async () => { - ({ testCookieName } = await setRandomCookie(contextId, stagehand)); - - const page = stagehand.page; - await page.goto("https://www.google.com", { - waitUntil: "domcontentloaded", - }); - await page.goBack(); - }); - - await test.step("Validate cookie persistence between pages", async () => { - const cookie = await findCookie(stagehand, testCookieName); - const found = !!cookie; - expect(found).toBe(true); - console.log("Cookie persisted between pages:", found); - - await stagehand.close(); - // Wait for context to persist - console.log("Waiting for context to persist..."); - await new Promise((resolve) => setTimeout(resolve, 5000)); - }); - - await test.step("Create another session with the same context", async () => { - // We don't need to persist cookies in this session, so we can mark persist=false - const newStagehand = new Stagehand({ - ...StagehandConfig, - browserbaseSessionCreateParams: { - projectId: BROWSERBASE_PROJECT_ID, - browserSettings: { - context: { - id: contextId, - persist: false, - }, - }, - }, - }); - await newStagehand.init(); - console.log( - `Reusing context ${contextId} during session ${newStagehand.browserbaseSessionID}`, - ); - const newPage = newStagehand.page; - await newPage.goto(CONTEXT_TEST_URL, { waitUntil: "domcontentloaded" }); - - const foundCookie = await findCookie(newStagehand, testCookieName); - console.log("Cookie found in new session:", !!foundCookie); - console.log( - "Cookie value matches:", - foundCookie?.value === testCookieValue, - ); - - await newStagehand.close(); - }); - }); -}); diff --git a/evals/deterministic/tests/browserbase/downloads.test.ts b/evals/deterministic/tests/browserbase/downloads.test.ts deleted file mode 100644 index 10366b6cd..000000000 --- a/evals/deterministic/tests/browserbase/downloads.test.ts +++ /dev/null @@ -1,69 +0,0 @@ -import { test, expect } from "@playwright/test"; -import AdmZip from "adm-zip"; -import StagehandConfig from "@/evals/deterministic/stagehand.config"; -import { Stagehand } from "@/dist"; -import Browserbase from "@browserbasehq/sdk"; - -const downloadRe = /sandstorm-(\d{13})+\.mp3/; - -test("Downloads", async () => { - const stagehand = new Stagehand(StagehandConfig); - await stagehand.init(); - const page = stagehand.page; - const context = stagehand.context; - - const client = await context.newCDPSession(page); - await client.send("Browser.setDownloadBehavior", { - behavior: "allow", - // `downloadPath` gets appended to the browser's default download directory. - // set to "downloads", it ends up being "/app/apps/browser/downloads/". - downloadPath: "downloads", - eventsEnabled: true, - }); - - await page.goto("https://browser-tests-alpha.vercel.app/api/download-test"); - - const [download] = await Promise.all([ - page.waitForEvent("download"), - page.locator("#download").click(), - ]); - - const downloadError = await download.failure(); - - await stagehand.close(); - - if (downloadError !== null) { - throw new Error( - `Download for session ${stagehand.browserbaseSessionID} failed: ${downloadError}`, - ); - } - - expect(async () => { - const bb = new Browserbase(); - const zipBuffer = await bb.sessions.downloads.list( - stagehand.browserbaseSessionID, - ); - if (!zipBuffer) { - throw new Error( - `Download buffer is empty for session ${stagehand.browserbaseSessionID}`, - ); - } - - const zip = new AdmZip(Buffer.from(await zipBuffer.arrayBuffer())); - const zipEntries = zip.getEntries(); - const mp3Entry = zipEntries.find((entry) => - downloadRe.test(entry.entryName), - ); - - if (!mp3Entry) { - throw new Error( - `Session ${stagehand.browserbaseSessionID} is missing a file matching "${downloadRe.toString()}" in its zip entries: ${JSON.stringify(zipEntries.map((entry) => entry.entryName))}`, - ); - } - - const expectedFileSize = 6137541; - expect(mp3Entry.header.size).toBe(expectedFileSize); - }).toPass({ - timeout: 30_000, - }); -}); diff --git a/evals/deterministic/tests/browserbase/sessions.test.ts b/evals/deterministic/tests/browserbase/sessions.test.ts deleted file mode 100644 index 67c97659c..000000000 --- a/evals/deterministic/tests/browserbase/sessions.test.ts +++ /dev/null @@ -1,67 +0,0 @@ -import { test, expect } from "@playwright/test"; -import { Stagehand } from "@/dist"; -import StagehandConfig from "@/evals/deterministic/stagehand.config"; -import Browserbase from "@browserbasehq/sdk"; - -test.describe("Browserbase Sessions", () => { - let browserbase: Browserbase; - let sessionId: string; - let bigStagehand: Stagehand; - - test.beforeAll(async () => { - browserbase = new Browserbase({ - apiKey: process.env.BROWSERBASE_API_KEY, - }); - bigStagehand = new Stagehand({ - ...StagehandConfig, - env: "BROWSERBASE", - browserbaseSessionCreateParams: { - projectId: process.env.BROWSERBASE_PROJECT_ID, - keepAlive: true, - }, - }); - await bigStagehand.init(); - await bigStagehand.page.goto( - "https://docs.stagehand.dev/get_started/introduction", - ); - sessionId = bigStagehand.browserbaseSessionID; - if (!sessionId) { - throw new Error("Failed to get browserbase session ID"); - } - }); - test.afterAll(async () => { - await bigStagehand.close(); - }); - test("resumes a session via sessionId", async () => { - const stagehand = new Stagehand({ - ...StagehandConfig, - env: "BROWSERBASE", - browserbaseSessionID: sessionId, - }); - await stagehand.init(); - - const page = stagehand.page; - - expect(page.url()).toBe( - "https://docs.stagehand.dev/get_started/introduction", - ); - await stagehand.close(); - }); - test("resumes a session via CDP URL", async () => { - const session = await browserbase.sessions.retrieve(sessionId); - const stagehand = new Stagehand({ - ...StagehandConfig, - env: "LOCAL", - localBrowserLaunchOptions: { - headless: true, - cdpUrl: session.connectUrl, - }, - }); - await stagehand.init(); - const page = stagehand.page; - - expect(page.url()).toBe( - "https://docs.stagehand.dev/get_started/introduction", - ); - }); -}); diff --git a/evals/deterministic/tests/browserbase/uploads.test.ts b/evals/deterministic/tests/browserbase/uploads.test.ts deleted file mode 100644 index 06b724423..000000000 --- a/evals/deterministic/tests/browserbase/uploads.test.ts +++ /dev/null @@ -1,36 +0,0 @@ -import { join } from "node:path"; -import { test, expect } from "@playwright/test"; -import { Stagehand } from "@/dist"; -import StagehandConfig from "@/evals/deterministic/stagehand.config"; - -test.describe("Playwright Upload", () => { - let stagehand: Stagehand; - - test.beforeAll(async () => { - stagehand = new Stagehand(StagehandConfig); - await stagehand.init(); - }); - - test.afterAll(async () => { - await stagehand.close(); - }); - - test("uploads a file", async () => { - const page = stagehand.page; - await page.goto("https://browser-tests-alpha.vercel.app/api/upload-test"); - - const fileInput = page.locator("#fileUpload"); - await fileInput.setInputFiles( - join(__dirname, "../..", "auxiliary", "logo.png"), - ); - - const fileNameSpan = page.locator("#fileName"); - const fileName = await fileNameSpan.innerText(); - - const fileSizeSpan = page.locator("#fileSize"); - const fileSize = Number(await fileSizeSpan.innerText()); - - expect(fileName).toBe("logo.png"); - expect(fileSize).toBeGreaterThan(0); - }); -}); diff --git a/evals/deterministic/tests/local/create.test.ts b/evals/deterministic/tests/local/create.test.ts deleted file mode 100644 index 48f78f4cb..000000000 --- a/evals/deterministic/tests/local/create.test.ts +++ /dev/null @@ -1,185 +0,0 @@ -import { test, expect } from "@playwright/test"; -import { Stagehand } from "@/dist"; -import path from "path"; -import fs from "fs"; -import os from "os"; -import type { Cookie } from "@playwright/test"; -import StagehandConfig from "../../stagehand.config"; - -test.describe("Local browser launch options", () => { - test("launches with default options when no localBrowserLaunchOptions provided", async () => { - const stagehand = new Stagehand(StagehandConfig); - await stagehand.init(); - - const context = stagehand.context; - expect(context.browser()).toBeDefined(); - expect(context.pages().length).toBe(1); - - await stagehand.close(); - }); - - test("respects custom userDataDir", async () => { - const customUserDataDir = path.join(os.tmpdir(), "custom-user-data"); - - const stagehand = new Stagehand({ - ...StagehandConfig, - localBrowserLaunchOptions: { - userDataDir: customUserDataDir, - headless: true, - }, - }); - await stagehand.init(); - - expect(fs.existsSync(customUserDataDir)).toBeTruthy(); - - await stagehand.close(); - - // Cleanup - fs.rmSync(customUserDataDir, { recursive: true, force: true }); - }); - - test("applies custom viewport settings", async () => { - const customViewport = { width: 1920, height: 1080 }; - - const stagehand = new Stagehand({ - ...StagehandConfig, - localBrowserLaunchOptions: { - ...StagehandConfig.localBrowserLaunchOptions, - viewport: customViewport, - }, - }); - await stagehand.init(); - - const page = await stagehand.context.newPage(); - const viewport = page.viewportSize(); - - expect(viewport).toEqual(customViewport); - - await stagehand.close(); - }); - - test("applies custom cookies", async () => { - const testCookies: Cookie[] = [ - { - name: "testCookie", - value: "testValue", - domain: "example.com", - path: "/", - expires: -1, - httpOnly: false, - secure: false, - sameSite: "Lax" as const, - }, - ]; - - const stagehand = new Stagehand({ - ...StagehandConfig, - localBrowserLaunchOptions: { - ...StagehandConfig.localBrowserLaunchOptions, - cookies: testCookies, - }, - }); - await stagehand.init(); - - const page = await stagehand.context.newPage(); - await page.goto("https://example.com"); - const cookies = await stagehand.context.cookies(); - - expect(cookies[0]).toMatchObject( - testCookies[0] as unknown as Record, - ); - - await stagehand.close(); - }); - - test("applies custom geolocation settings", async () => { - const customGeolocation = { - latitude: 40.7128, - longitude: -74.006, - }; - - const stagehand = new Stagehand({ - ...StagehandConfig, - localBrowserLaunchOptions: { - ...StagehandConfig.localBrowserLaunchOptions, - geolocation: customGeolocation, - permissions: ["geolocation"], - }, - }); - await stagehand.init(); - - const page = await stagehand.context.newPage(); - await page.goto("https://example.com"); - - const location = await page.evaluate(() => { - return new Promise((resolve) => { - navigator.geolocation.getCurrentPosition( - (position) => { - resolve({ - latitude: position.coords.latitude, - longitude: position.coords.longitude, - }); - }, - () => resolve(null), - ); - }); - }); - - expect(location).toEqual(customGeolocation); - - await stagehand.close(); - }); - - test("applies custom timezone and locale", async () => { - const stagehand = new Stagehand({ - ...StagehandConfig, - localBrowserLaunchOptions: { - ...StagehandConfig.localBrowserLaunchOptions, - locale: "ja-JP", - timezoneId: "Asia/Tokyo", - }, - }); - await stagehand.init(); - - const page = await stagehand.context.newPage(); - await page.goto("https://example.com"); - - const { locale, timezone } = await page.evaluate(() => ({ - locale: navigator.language, - timezone: Intl.DateTimeFormat().resolvedOptions().timeZone, - })); - - expect(locale).toBe("ja-JP"); - expect(timezone).toBe("Asia/Tokyo"); - - await stagehand.close(); - }); - - test("records video when enabled", async () => { - const videoDir = path.join(os.tmpdir(), "test-videos"); - fs.mkdirSync(videoDir, { recursive: true }); - - const stagehand = new Stagehand({ - ...StagehandConfig, - localBrowserLaunchOptions: { - ...StagehandConfig.localBrowserLaunchOptions, - recordVideo: { - dir: videoDir, - size: { width: 800, height: 600 }, - }, - }, - }); - await stagehand.init(); - - const page = await stagehand.context.newPage(); - await page.goto("https://example.com"); - await stagehand.close(); - - const videos = fs.readdirSync(videoDir); - expect(videos.length).toBeGreaterThan(0); - expect(videos[0]).toMatch(/\.webm$/); - - // Cleanup - fs.rmSync(videoDir, { recursive: true, force: true }); - }); -}); diff --git a/evals/deterministic/tests/page/addInitScript.test.ts b/evals/deterministic/tests/page/addInitScript.test.ts deleted file mode 100644 index 56be8ef41..000000000 --- a/evals/deterministic/tests/page/addInitScript.test.ts +++ /dev/null @@ -1,40 +0,0 @@ -import { test, expect } from "@playwright/test"; -import { Stagehand } from "@/dist"; -import StagehandConfig from "@/evals/deterministic/stagehand.config"; - -test.describe("StagehandPage - addInitScript", () => { - test("should inject a script before the page loads", async () => { - const stagehand = new Stagehand(StagehandConfig); - await stagehand.init(); - - const page = stagehand.page; - - await page.addInitScript(() => { - const w = window as typeof window & { - __testInitScriptVar?: string; - }; - w.__testInitScriptVar = "Hello from init script!"; - }); - - await page.goto("https://example.com"); - - const result = await page.evaluate(() => { - const w = window as typeof window & { - __testInitScriptVar?: string; - }; - return w.__testInitScriptVar; - }); - expect(result).toBe("Hello from init script!"); - - await page.goto("https://docs.browserbase.com/"); - const resultAfterNavigation = await page.evaluate(() => { - const w = window as typeof window & { - __testInitScriptVar?: string; - }; - return w.__testInitScriptVar; - }); - expect(resultAfterNavigation).toBe("Hello from init script!"); - - await stagehand.close(); - }); -}); diff --git a/evals/deterministic/tests/page/addRemoveLocatorHandler.test.ts b/evals/deterministic/tests/page/addRemoveLocatorHandler.test.ts deleted file mode 100644 index a46abc92d..000000000 --- a/evals/deterministic/tests/page/addRemoveLocatorHandler.test.ts +++ /dev/null @@ -1,93 +0,0 @@ -import { test, expect } from "@playwright/test"; -import { Stagehand } from "@/dist"; -import StagehandConfig from "@/evals/deterministic/stagehand.config"; - -test.describe("StagehandPage - addLocatorHandler and removeLocatorHandler", () => { - // This HTML snippet is reused by both tests. - // The "Sign up to the newsletter" overlay appears after 2 seconds. - // The "No thanks" button hides it. - const overlayHTML = ` - - - - - - - - `; - - test("should use a custom locator handler to dismiss the overlay", async () => { - const stagehand = new Stagehand(StagehandConfig); - await stagehand.init(); - - const { page } = stagehand; - - await page.addLocatorHandler( - page.getByText("Sign up to the newsletter"), - async () => { - console.log("Overlay detected. Clicking 'No thanks' to remove it..."); - await page.getByRole("button", { name: "No thanks" }).click(); - }, - ); - - await page.goto("https://example.com"); - await page.setContent(overlayHTML); - - await page.waitForTimeout(5000); - - await page.getByRole("button", { name: "Start here" }).click(); - - const isOverlayVisible = await page - .getByText("Sign up to the newsletter") - .isVisible() - .catch(() => false); - - await stagehand.close(); - - expect(isOverlayVisible).toBeFalsy(); - }); - - test("should remove a custom locator handler so overlay stays visible", async () => { - const stagehand = new Stagehand(StagehandConfig); - await stagehand.init(); - - const { page } = stagehand; - - const locator = page.getByText("Sign up to the newsletter"); - await page.addLocatorHandler(locator, async () => { - console.log("Overlay detected. Clicking 'No thanks' to remove it..."); - await page.getByRole("button", { name: "No thanks" }).click(); - }); - - await page.removeLocatorHandler(locator); - console.log("Locator handler removed — overlay will not be dismissed now."); - - await page.goto("https://example.com"); - await page.setContent(overlayHTML); - - await page.waitForTimeout(5000); - - await page.getByRole("button", { name: "Start here" }).click(); - - const isOverlayVisible = await page - .getByText("Sign up to the newsletter") - .isVisible() - .catch(() => false); - - await stagehand.close(); - expect(isOverlayVisible).toBe(true); - }); -}); diff --git a/evals/deterministic/tests/page/addTags.test.ts b/evals/deterministic/tests/page/addTags.test.ts deleted file mode 100644 index 463d109a2..000000000 --- a/evals/deterministic/tests/page/addTags.test.ts +++ /dev/null @@ -1,79 +0,0 @@ -import { test, expect } from "@playwright/test"; -import { Stagehand } from "@/dist"; -import StagehandConfig from "@/evals/deterministic/stagehand.config"; - -test.describe("StagehandPage - addScriptTag and addStyleTag", () => { - let stagehand: Stagehand; - - test.beforeAll(async () => { - stagehand = new Stagehand(StagehandConfig); - await stagehand.init(); - }); - - test.afterAll(async () => { - await stagehand.close(); - }); - - test("should inject a script tag and have access to the defined function", async () => { - const { page } = stagehand; - - await page.setContent(` - - -

    Hello, world!

    - - - `); - - await page.addScriptTag({ - content: ` - window.sayHello = function() { - document.getElementById("greeting").textContent = "Hello from injected script!"; - } - `, - }); - - await page.evaluate(() => { - const w = window as typeof window & { - sayHello?: () => void; - }; - w.sayHello?.(); - }); - - const text = await page.locator("#greeting").textContent(); - expect(text).toBe("Hello from injected script!"); - }); - - test("should inject a style tag and apply styles", async () => { - const { page } = stagehand; - - await page.setContent(` - - -
    Some text
    - - - `); - - await page.addStyleTag({ - content: ` - #styledDiv { - color: red; - font-weight: bold; - } - `, - }); - - const color = await page.evaluate(() => { - const el = document.getElementById("styledDiv"); - return window.getComputedStyle(el!).color; - }); - expect(color).toBe("rgb(255, 0, 0)"); - - const fontWeight = await page.evaluate(() => { - const el = document.getElementById("styledDiv"); - return window.getComputedStyle(el!).fontWeight; - }); - expect(["bold", "700"]).toContain(fontWeight); - }); -}); diff --git a/evals/deterministic/tests/page/bringToFront.test.ts b/evals/deterministic/tests/page/bringToFront.test.ts deleted file mode 100644 index d25ddb6c3..000000000 --- a/evals/deterministic/tests/page/bringToFront.test.ts +++ /dev/null @@ -1,37 +0,0 @@ -import { test, expect } from "@playwright/test"; -import { Stagehand } from "@/dist"; -import StagehandConfig from "@/evals/deterministic/stagehand.config"; - -test.describe("StagehandPage - bringToFront", () => { - test("should bring a background page to the front and allow further actions", async () => { - const stagehand = new Stagehand(StagehandConfig); - await stagehand.init(); - - const { page: page1 } = stagehand; - - const page2 = await stagehand.context.newPage(); - await page2.goto("https://example.com"); - const page2Title = await page2.title(); - console.log("Page2 Title:", page2Title); - - await page1.goto("https://www.google.com"); - const page1TitleBefore = await page1.title(); - console.log("Page1 Title before:", page1TitleBefore); - - await page1.bringToFront(); - - await page1.goto("https://docs.browserbase.com"); - const page1TitleAfter = await page1.title(); - console.log("Page1 Title after:", page1TitleAfter); - - await page2.bringToFront(); - const page2URLBefore = page2.url(); - console.log("Page2 URL before navigation:", page2URLBefore); - - await stagehand.close(); - - expect(page1TitleBefore).toContain("Google"); - expect(page1TitleAfter).toContain("Browserbase"); - expect(page2Title).toContain("Example Domain"); - }); -}); diff --git a/evals/deterministic/tests/page/content.test.ts b/evals/deterministic/tests/page/content.test.ts deleted file mode 100644 index fe95aae69..000000000 --- a/evals/deterministic/tests/page/content.test.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { test, expect } from "@playwright/test"; -import { Stagehand } from "@/dist"; -import StagehandConfig from "@/evals/deterministic/stagehand.config"; - -test.describe("StagehandPage - content", () => { - test("should retrieve the full HTML content of the page", async () => { - const stagehand = new Stagehand(StagehandConfig); - await stagehand.init(); - - const page = stagehand.page; - await page.goto("https://example.com"); - const html = await page.content(); - expect(html).toContain("Example Domain"); - expect(html).toContain("

    Example Domain

    "); - - await stagehand.close(); - }); -}); diff --git a/evals/deterministic/tests/page/evaluate.test.ts b/evals/deterministic/tests/page/evaluate.test.ts deleted file mode 100644 index 2610735be..000000000 --- a/evals/deterministic/tests/page/evaluate.test.ts +++ /dev/null @@ -1,31 +0,0 @@ -import { test, expect } from "@playwright/test"; -import { Stagehand } from "@/dist"; -import StagehandConfig from "@/evals/deterministic/stagehand.config"; - -test.describe("StagehandPage - JavaScript Evaluation", () => { - test("can evaluate JavaScript in the page context", async () => { - const stagehand = new Stagehand(StagehandConfig); - await stagehand.init(); - - const page = stagehand.page; - - await page.goto("https://example.com"); - - const sum = await page.evaluate(() => 2 + 2); - expect(sum).toBe(4); - - const pageTitle = await page.evaluate(() => document.title); - expect(pageTitle).toMatch(/example/i); - - const obj = await page.evaluate(() => { - return { - message: "Hello from the browser", - userAgent: navigator.userAgent, - }; - }); - expect(obj).toHaveProperty("message", "Hello from the browser"); - expect(obj.userAgent).toBeDefined(); - - await stagehand.close(); - }); -}); diff --git a/evals/deterministic/tests/page/expose.test.ts b/evals/deterministic/tests/page/expose.test.ts deleted file mode 100644 index 9572b39c0..000000000 --- a/evals/deterministic/tests/page/expose.test.ts +++ /dev/null @@ -1,63 +0,0 @@ -import { test, expect } from "@playwright/test"; -import { Stagehand } from "@/dist"; -import StagehandConfig from "@/evals/deterministic/stagehand.config"; - -test.describe("StagehandPage - evaluateHandle, exposeBinding, exposeFunction", () => { - let stagehand: Stagehand; - - test.beforeAll(async () => { - stagehand = new Stagehand(StagehandConfig); - await stagehand.init(); - }); - - test.afterAll(async () => { - await stagehand.close(); - }); - - test("demonstrates evaluateHandle, exposeBinding, and exposeFunction", async () => { - const { page } = stagehand; - - await page.setContent(` - - -
    Initial Text
    - - - `); - - const divHandle = await page.evaluateHandle(() => { - return document.getElementById("myDiv"); - }); - await divHandle.evaluate((div, newText) => { - div.textContent = newText; - }, "Text updated via evaluateHandle"); - - const text = await page.locator("#myDiv").textContent(); - expect(text).toBe("Text updated via evaluateHandle"); - - await page.exposeBinding("myBinding", async (source, arg: string) => { - console.log("myBinding called from page with arg:", arg); - return `Node responded with: I got your message: "${arg}"`; - }); - - const responseFromBinding = await page.evaluate(async () => { - const w = window as typeof window & { - myBinding?: (arg: string) => Promise; - }; - return w.myBinding?.("Hello from the browser"); - }); - expect(responseFromBinding).toMatch(/I got your message/); - - await page.exposeFunction("addNumbers", (a: number, b: number) => { - return a + b; - }); - - const sum = await page.evaluate(async () => { - const w = window as typeof window & { - addNumbers?: (a: number, b: number) => number; - }; - return w.addNumbers?.(3, 7); - }); - expect(sum).toBe(10); - }); -}); diff --git a/evals/deterministic/tests/page/frames.test.ts b/evals/deterministic/tests/page/frames.test.ts deleted file mode 100644 index 18c50942c..000000000 --- a/evals/deterministic/tests/page/frames.test.ts +++ /dev/null @@ -1,66 +0,0 @@ -import { test, expect } from "@playwright/test"; -import { Stagehand } from "@/dist"; -import StagehandConfig from "@/evals/deterministic/stagehand.config"; - -test.describe("StagehandPage - frame operations", () => { - let stagehand: Stagehand; - - test.beforeAll(async () => { - stagehand = new Stagehand(StagehandConfig); - await stagehand.init(); - }); - - test.afterAll(async () => { - await stagehand.close(); - }); - - test("should use page.mainFrame(), page.frames(), page.frame(), and page.frameLocator()", async () => { - const { page } = stagehand; - - await page.setContent(` - - - - - - - - `); - - await page.waitForSelector('iframe[name="frame-one"]'); - await page.waitForSelector('iframe[name="frame-two"]'); - - const frames = page.frames(); - console.log( - "All frames found:", - frames.map((f) => f.name()), - ); - expect(frames).toHaveLength(3); - - const mainFrame = page.mainFrame(); - console.log("Main frame name:", mainFrame.name()); - expect(mainFrame.name()).toBe(""); - - const frameOne = page.frame({ name: "frame-one" }); - expect(frameOne).not.toBeNull(); - - const frameOneText = await frameOne?.locator("h1").textContent(); - expect(frameOneText).toBe("Hello from Frame 1"); - - const frameTwoLocator = page.frameLocator("iframe[name='frame-two']"); - const frameTwoText = await frameTwoLocator.locator("h1").textContent(); - expect(frameTwoText).toBe("Hello from Frame 2"); - - const frameTwo = page.frame({ name: "frame-two" }); - expect(frameTwo).not.toBeNull(); - - const frameTwoTextAgain = await frameTwo?.locator("h1").textContent(); - expect(frameTwoTextAgain).toBe("Hello from Frame 2"); - }); -}); diff --git a/evals/deterministic/tests/page/getBy.test.ts b/evals/deterministic/tests/page/getBy.test.ts deleted file mode 100644 index 5da3a3976..000000000 --- a/evals/deterministic/tests/page/getBy.test.ts +++ /dev/null @@ -1,50 +0,0 @@ -import { test, expect } from "@playwright/test"; -import { Stagehand } from "@/dist"; -import StagehandConfig from "@/evals/deterministic/stagehand.config"; - -test.describe("StagehandPage - Built-in locators", () => { - let stagehand: Stagehand; - - test.beforeAll(async () => { - stagehand = new Stagehand(StagehandConfig); - await stagehand.init(); - }); - - test.afterAll(async () => { - await stagehand.close(); - }); - - test("demonstrates getByAltText, getByLabel, getByPlaceholder, getByRole, getByTestId, getByText, getByTitle", async () => { - const { page } = stagehand; - await page.setContent(` - - - Profile picture - - - - -
    Hello World!
    -

    This is some descriptive text on the page.

    -

    Site Title

    - - - `); - const image = page.getByAltText("Profile picture"); - await expect(image).toBeVisible(); - const usernameInput = page.getByLabel("Username"); - await expect(usernameInput).toBeVisible(); - const emailInput = page.getByPlaceholder("Enter your email"); - await expect(emailInput).toBeVisible(); - const signInButton = page.getByRole("button", { name: "Sign in" }); - await expect(signInButton).toBeVisible(); - const greetingDiv = page.getByTestId("greeting"); - await expect(greetingDiv).toHaveText("Hello World!"); - const descriptiveText = page.getByText( - "This is some descriptive text on the page.", - ); - await expect(descriptiveText).toBeVisible(); - const heading = page.getByTitle("A heading for the page"); - await expect(heading).toHaveText("Site Title"); - }); -}); diff --git a/evals/deterministic/tests/page/navigation.test.ts b/evals/deterministic/tests/page/navigation.test.ts deleted file mode 100644 index e9acb7a7c..000000000 --- a/evals/deterministic/tests/page/navigation.test.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { test, expect } from "@playwright/test"; -import { Stagehand } from "@/dist"; -import StagehandConfig from "@/evals/deterministic/stagehand.config"; - -test.describe("StagehandPage - Navigation", () => { - test("should navigate back and forward between pages", async () => { - const stagehand = new Stagehand(StagehandConfig); - await stagehand.init(); - - const page = stagehand.page; - - await page.goto("https://example.com"); - expect(page.url()).toBe("https://example.com/"); - - await page.goto("https://docs.browserbase.com/introduction"); - expect(page.url()).toBe("https://docs.browserbase.com/introduction"); - - await page.goBack(); - expect(page.url()).toBe("https://example.com/"); - - await page.goForward(); - expect(page.url()).toBe("https://docs.browserbase.com/introduction"); - - await stagehand.close(); - }); -}); diff --git a/evals/deterministic/tests/page/on.test.ts b/evals/deterministic/tests/page/on.test.ts deleted file mode 100644 index 97ec5bbfe..000000000 --- a/evals/deterministic/tests/page/on.test.ts +++ /dev/null @@ -1,134 +0,0 @@ -import { expect, test } from "@playwright/test"; -import { Stagehand } from "@/dist"; -import StagehandConfig from "@/evals/deterministic/stagehand.config"; - -test.describe("StagehandPage - page.on()", () => { - test("should click on the crewAI blog tab", async () => { - const stagehand = new Stagehand(StagehandConfig); - await stagehand.init(); - - const page = stagehand.page; - await page.goto( - "https://docs.browserbase.com/integrations/crew-ai/introduction", - ); - - let clickPromise: Promise; - - page.on("popup", async (newPage) => { - clickPromise = newPage.click( - "body > div.page-wrapper > div.navbar-2.w-nav > div.padding-global.top-bot > div > div.navigation-left > nav > a:nth-child(7)", - ); - }); - - await page.goto( - "https://docs.browserbase.com/integrations/crew-ai/introduction", - ); - - await page.click( - "#content-area > div.relative.mt-8.prose.prose-gray.dark\\:prose-invert > p:nth-child(2) > a", - ); - - await clickPromise; - - await stagehand.close(); - }); - - test("should close the new tab and navigate to it on the existing page", async () => { - const stagehand = new Stagehand(StagehandConfig); - await stagehand.init(); - - const page = stagehand.page; - await page.goto( - "https://docs.browserbase.com/integrations/crew-ai/introduction", - ); - - let navigatePromise: Promise; - - page.on("popup", async (newPage) => { - navigatePromise = Promise.allSettled([ - newPage.close(), - page.goto(newPage.url(), { waitUntil: "domcontentloaded" }), - ]); - }); - - // Click on the crewAI blog tab - await page.click( - "#content-area > div.relative.mt-8.prose.prose-gray.dark\\:prose-invert > p:nth-child(2) > a", - ); - - await navigatePromise; - - await page.click( - "body > div.page-wrapper > div.navbar-2.w-nav > div.padding-global.top-bot > div > div.navigation-left > nav > a:nth-child(3)", - ); - - await page.waitForLoadState("domcontentloaded"); - - const currentUrl = page.url(); - expect(currentUrl).toBe("https://www.crewai.com/open-source"); - - await stagehand.close(); - }); - - test("should handle console events", async () => { - const stagehand = new Stagehand(StagehandConfig); - await stagehand.init(); - - const page = stagehand.page; - await page.goto("https://example.com"); - - const messages: string[] = []; - page.on("console", (msg) => { - messages.push(msg.text()); - }); - - await page.evaluate(() => console.log("Test console log")); - - expect(messages).toContain("Test console log"); - - await stagehand.close(); - }); - - test("should handle dialog events", async () => { - const stagehand = new Stagehand(StagehandConfig); - await stagehand.init(); - - const page = stagehand.page; - await page.goto("https://example.com"); - - page.on("dialog", async (dialog) => { - expect(dialog.message()).toBe("Test alert"); - await dialog.dismiss(); - }); - - await page.evaluate(() => alert("Test alert")); - - await stagehand.close(); - }); - - test("should handle request and response events", async () => { - const stagehand = new Stagehand(StagehandConfig); - await stagehand.init(); - - const page = stagehand.page; - await page.goto("https://example.com"); - - const requests: string[] = []; - const responses: string[] = []; - - page.on("request", (request) => { - requests.push(request.url()); - }); - - page.on("response", (response) => { - responses.push(response.url()); - }); - - await page.goto("https://example.com"); - - expect(requests).toContain("https://example.com/"); - expect(responses).toContain("https://example.com/"); - - await stagehand.close(); - }); -}); diff --git a/evals/deterministic/tests/page/pageContext.test.ts b/evals/deterministic/tests/page/pageContext.test.ts deleted file mode 100644 index a0f36911c..000000000 --- a/evals/deterministic/tests/page/pageContext.test.ts +++ /dev/null @@ -1,59 +0,0 @@ -import { test, expect } from "@playwright/test"; -import { Stagehand } from "@/dist"; -import StagehandConfig from "@/evals/deterministic/stagehand.config"; - -test.describe("StagehandPage - page.context()", () => { - let stagehand: Stagehand; - - test.beforeEach(async () => { - stagehand = new Stagehand(StagehandConfig); - await stagehand.init(); - }); - - test.afterEach(async () => { - if (stagehand) { - try { - await stagehand.close(); - } catch (error) { - console.error("[afterEach] Error during stagehand.close():", error); - } - } else { - console.log("[afterEach] Stagehand was not defined, skipping close()."); - } - }); - - test("should confirm page.context() and stagehand.context share state", async () => { - const page = stagehand.page; - const stagehandContext = stagehand.context; - const pageContext = page.context(); - - await pageContext.addCookies([ - { - name: "stagehandTestCookie", - value: "hello-stagehand", - domain: "example.com", - path: "/", - expires: Math.floor(Date.now() / 1000) + 3600, // 1 hour - httpOnly: false, - secure: false, - sameSite: "Lax", - }, - ]); - - const cookies = await stagehandContext.cookies("https://example.com"); - - const testCookie = cookies.find((c) => c.name === "stagehandTestCookie"); - expect(testCookie).toBeDefined(); - expect(testCookie?.value).toBe("hello-stagehand"); - - const extraPage = await pageContext.newPage(); - await extraPage.goto("https://example.com"); - const contextPages = stagehandContext.pages(); - - // The newly created page should be recognized by stagehandContext as well. - const foundExtraPage = contextPages.find( - (p) => p.url() === "https://example.com/", - ); - expect(foundExtraPage).toBeDefined(); - }); -}); diff --git a/evals/deterministic/tests/page/reload.test.ts b/evals/deterministic/tests/page/reload.test.ts deleted file mode 100644 index 31518a747..000000000 --- a/evals/deterministic/tests/page/reload.test.ts +++ /dev/null @@ -1,40 +0,0 @@ -import { test, expect } from "@playwright/test"; -import { Stagehand } from "@/dist"; -import StagehandConfig from "@/evals/deterministic/stagehand.config"; - -test.describe("StagehandPage - Reload", () => { - test("should reload the page and reset page state", async () => { - const stagehand = new Stagehand(StagehandConfig); - await stagehand.init(); - - const page = stagehand.page; - await page.goto("https://docs.browserbase.com/"); - - await page.evaluate(() => { - const w = window as typeof window & { - __testReloadMarker?: string; - }; - w.__testReloadMarker = "Hello Reload!"; - }); - - const markerBeforeReload = await page.evaluate(() => { - const w = window as typeof window & { - __testReloadMarker?: string; - }; - return w.__testReloadMarker; - }); - expect(markerBeforeReload).toBe("Hello Reload!"); - - await page.reload(); - - const markerAfterReload = await page.evaluate(() => { - const w = window as typeof window & { - __testReloadMarker?: string; - }; - return w.__testReloadMarker; - }); - expect(markerAfterReload).toBeUndefined(); - - await stagehand.close(); - }); -}); diff --git a/evals/deterministic/tests/page/waitFor.test.ts b/evals/deterministic/tests/page/waitFor.test.ts deleted file mode 100644 index e42dd3b58..000000000 --- a/evals/deterministic/tests/page/waitFor.test.ts +++ /dev/null @@ -1,165 +0,0 @@ -import { test, expect } from "@playwright/test"; -import { Stagehand } from "@/dist"; -import StagehandConfig from "@/evals/deterministic/stagehand.config"; - -test.describe("StagehandPage - waitFor", () => { - test("should wait for an element to become visible", async () => { - const stagehand = new Stagehand(StagehandConfig); - await stagehand.init(); - - const page = stagehand.page; - await page.goto("https://docs.browserbase.com/introduction"); - const dynamicElement = page.locator( - "div.grid:nth-child(1) > a:nth-child(1) > div:nth-child(1)", - ); - - const isVisibleBefore = await dynamicElement.isVisible(); - expect(isVisibleBefore).toBe(false); - - const clickableElement = page.locator( - "div.not-prose:nth-child(2) > a:nth-child(1) > div:nth-child(1)", - ); - await clickableElement.click(); - - await dynamicElement.waitFor({ state: "visible" }); - - const isVisibleAfter = await dynamicElement.isVisible(); - expect(isVisibleAfter).toBe(true); - - await stagehand.close(); - }); - - test("should wait for an element to be detached", async () => { - const stagehand = new Stagehand(StagehandConfig); - await stagehand.init(); - - const page = stagehand.page; - await page.goto("https://docs.browserbase.com/introduction"); - - const disappearingElement = page.locator( - "div.not-prose:nth-child(2) > a:nth-child(1) > div:nth-child(1)", - ); - - await disappearingElement.click(); - await disappearingElement.waitFor({ state: "detached" }); - - const isAttachedAfter = await disappearingElement.isVisible(); - expect(isAttachedAfter).toBe(false); - - await stagehand.close(); - }); - - test("should wait for a specific event (waitForEvent)", async () => { - const stagehand = new Stagehand(StagehandConfig); - await stagehand.init(); - - const page = stagehand.page; - await page.goto("https://docs.browserbase.com/introduction"); - - const consolePromise = page.waitForEvent("console"); - await page.evaluate(() => { - console.log("Hello from the browser console!"); - }); - const consoleMessage = await consolePromise; - expect(consoleMessage.text()).toBe("Hello from the browser console!"); - - await stagehand.close(); - }); - - test("should wait for a function to return true (waitForFunction)", async () => { - const stagehand = new Stagehand(StagehandConfig); - await stagehand.init(); - - const page = stagehand.page; - await page.goto("https://docs.browserbase.com/introduction"); - - await page.evaluate(() => { - setTimeout(() => { - const w = window as typeof window & { - __stagehandFlag?: boolean; - }; - w.__stagehandFlag = true; - }, 1000); - }); - - await page.waitForFunction(() => { - const w = window as typeof window & { - __stagehandFlag?: boolean; - }; - return w.__stagehandFlag === true; - }); - - const value = await page.evaluate(() => { - const w = window as typeof window & { - __stagehandFlag?: boolean; - }; - return w.__stagehandFlag; - }); - expect(value).toBe(true); - - await stagehand.close(); - }); - - test("should wait for the load state (waitForLoadState)", async () => { - const stagehand = new Stagehand(StagehandConfig); - await stagehand.init(); - - const page = stagehand.page; - await page.goto("https://docs.browserbase.com/introduction"); - await page.waitForLoadState("networkidle"); - const heroTitle = page.locator("h1"); - await expect(heroTitle).toHaveText(/Documentation/i); - - await stagehand.close(); - }); - - test("should wait for a specific request (waitForRequest)", async () => { - const stagehand = new Stagehand(StagehandConfig); - await stagehand.init(); - - const page = stagehand.page; - const requestPromise = page.waitForRequest((req) => - req.url().includes("mintlify"), - ); - - await page.goto("https://docs.browserbase.com/introduction"); - const matchingRequest = await requestPromise; - expect(matchingRequest.url()).toContain("mintlify"); - - await stagehand.close(); - }); - - test("should wait for a specific response (waitForResponse)", async () => { - const stagehand = new Stagehand(StagehandConfig); - await stagehand.init(); - - const page = stagehand.page; - const responsePromise = page.waitForResponse( - (res) => res.url().includes("introduction") && res.status() === 200, - ); - - await page.goto("https://docs.browserbase.com/introduction"); - const matchingResponse = await responsePromise; - expect(await matchingResponse.text()).toContain("Browserbase"); - - await stagehand.close(); - }); - - test("should wait for a URL (waitForURL)", async () => { - const stagehand = new Stagehand(StagehandConfig); - await stagehand.init(); - - const page = stagehand.page; - await page.goto("https://docs.browserbase.com"); - - const getStartedLink = page.locator( - "div.not-prose:nth-child(3) > a:nth-child(1) > div:nth-child(1)", - ); - await getStartedLink.click(); - - await page.waitForURL(/.*getting-started.*/); - expect(page.url()).toContain("/getting-started"); - - await stagehand.close(); - }); -}); diff --git a/evals/env.ts b/evals/env.ts deleted file mode 100644 index 45f877bd1..000000000 --- a/evals/env.ts +++ /dev/null @@ -1,18 +0,0 @@ -/** - * Determine the current environment in which the evaluations are running: - * - BROWSERBASE or LOCAL - * - * The environment is read from the EVAL_ENV environment variable. - */ -export const env: "BROWSERBASE" | "LOCAL" = - process.env.EVAL_ENV?.toLowerCase() === "browserbase" - ? "BROWSERBASE" - : "LOCAL"; - -/** - * Enable or disable caching based on the EVAL_ENABLE_CACHING environment variable. - * Caching may improve performance by not re-fetching or re-computing certain results. - * By default, caching is disabled unless explicitly enabled. - */ -export const enableCaching = - process.env.EVAL_ENABLE_CACHING?.toLowerCase() === "true"; diff --git a/evals/evals.config.json b/evals/evals.config.json deleted file mode 100644 index ade809c32..000000000 --- a/evals/evals.config.json +++ /dev/null @@ -1,322 +0,0 @@ -{ - "tasks": [ - { - "name": "history", - "categories": ["combination"] - }, - { - "name": "expect_act_timeout", - "categories": ["regression"] - }, - { - "name": "extract_repo_name", - "categories": ["extract"] - }, - { - "name": "amazon_add_to_cart", - "categories": ["act"] - }, - { - "name": "instructions", - "categories": ["regression", "combination"] - }, - { - "name": "bidnet", - "categories": ["act"] - }, - { - "name": "ionwave", - "categories": ["act", "regression"], - "extract_method": "domExtract" - }, - { - "name": "nonsense_action", - "categories": ["act"] - }, - { - "name": "peeler_simple", - "categories": ["act"] - }, - { - "name": "simple_google_search", - "categories": ["act"] - }, - { - "name": "vantechjournal", - "categories": ["act"] - }, - { - "name": "wikipedia", - "categories": ["act"] - }, - - { - "name": "allrecipes", - "categories": ["combination"] - }, - { - "name": "arxiv", - "categories": ["combination"] - }, - { - "name": "extract_collaborators", - "categories": ["combination"] - }, - { - "name": "extract_github_commits", - "categories": ["combination"] - }, - { - "name": "imdb_movie_details", - "categories": ["combination"] - }, - { - "name": "peeler_complex", - "categories": ["combination"] - }, - { - "name": "sciquest", - "categories": ["combination"] - }, - { - "name": "wichita", - "categories": ["combination", "regression"], - "extract_method": "domExtract" - }, - { - "name": "hn_aisdk", - "categories": ["llm_clients"] - }, - { - "name": "hn_langchain", - "categories": ["llm_clients"] - }, - { - "name": "hn_customOpenAI", - "categories": ["llm_clients"] - }, - { - "name": "apple", - "categories": ["experimental"] - }, - { - "name": "combination_sauce", - "categories": ["experimental"] - }, - { - "name": "costar", - "categories": ["experimental"] - }, - { - "name": "expedia", - "categories": ["experimental"] - }, - { - "name": "expedia_search", - "categories": ["experimental"] - }, - { - "name": "extract_aigrant_companies", - "categories": ["text_extract", "regression"], - "extract_method": "textExtract" - }, - { - "name": "extract_capacitor_info", - "categories": ["experimental", "text_extract"] - }, - { - "name": "extract_partners", - "categories": ["experimental"] - }, - { - "name": "extract_press_releases", - "categories": ["experimental", "text_extract"] - }, - { - "name": "extract_snowshoeing_destinations", - "categories": ["experimental", "text_extract"] - }, - { - "name": "google_jobs", - "categories": ["experimental"] - }, - { - "name": "homedepot", - "categories": ["experimental"] - }, - { - "name": "rakuten_jp", - "categories": ["experimental"] - }, - { - "name": "stock_x", - "categories": ["experimental"] - }, - { - "name": "ted_talk", - "categories": ["experimental"] - }, - - { - "name": "extract_baptist_health", - "categories": ["extract"] - }, - { - "name": "extract_github_stars", - "categories": ["extract"] - }, - { - "name": "extract_memorial_healthcare", - "categories": ["extract", "regression"], - "extract_method": "domExtract" - }, - { - "name": "extract_nhl_stats", - "categories": ["extract"] - }, - { - "name": "extract_professional_info", - "categories": ["extract"] - }, - { - "name": "extract_csa", - "categories": ["text_extract"] - }, - { - "name": "extract_resistor_info", - "categories": ["extract"] - }, - { - "name": "extract_rockauto", - "categories": ["extract"] - }, - { - "name": "extract_staff_members", - "categories": ["extract"] - }, - - { - "name": "ionwave_observe", - "categories": ["observe"] - }, - { - "name": "panamcs", - "categories": ["observe"] - }, - { - "name": "vanta_h", - "categories": ["experimental"] - }, - { - "name": "extract_area_codes", - "categories": ["text_extract"] - }, - { - "name": "extract_public_notices", - "categories": ["text_extract"] - }, - { - "name": "extract_jstor_news", - "categories": ["text_extract"] - }, - { - "name": "extract_apartments", - "categories": ["text_extract"] - }, - { - "name": "extract_zillow", - "categories": ["text_extract"] - }, - { - "name": "observe_github", - "categories": ["observe", "regression"], - "extract_method": "textExtract" - }, - { - "name": "observe_vantechjournal", - "categories": ["observe", "regression"], - "extract_method": "textExtract" - }, - { - "name": "observe_amazon_add_to_cart", - "categories": ["observe"] - }, - { - "name": "observe_simple_google_search", - "categories": ["observe"] - }, - { - "name": "observe_yc_startup", - "categories": ["observe"] - }, - { - "name": "observe_taxes", - "categories": ["observe"] - }, - { - "name": "observe_iframes1", - "categories": ["regression", "observe"] - }, - { - "name": "observe_iframes2", - "categories": ["regression", "observe"] - }, - { - "name": "extract_hamilton_weather", - "categories": ["targeted_extract", "regression"], - "extract_method": "textExtract" - }, - { - "name": "extract_regulations_table", - "categories": ["targeted_extract"] - }, - { - "name": "extract_recipe", - "categories": ["targeted_extract"] - }, - { - "name": "extract_aigrant_targeted", - "categories": ["targeted_extract"] - }, - { - "name": "extract_aigrant_targeted_2", - "categories": ["targeted_extract"] - }, - { - "name": "extract_geniusee", - "categories": ["targeted_extract"] - }, - { - "name": "extract_geniusee_2", - "categories": ["targeted_extract"] - }, - { - "name": "scroll_50", - "categories": ["regression", "act"] - }, - { - "name": "scroll_75", - "categories": ["regression", "act"] - }, - { - "name": "nextChunk", - "categories": ["regression", "act"] - }, - { - "name": "prevChunk", - "categories": ["regression", "act"] - }, - { - "name": "google_flights", - "categories": ["act"] - }, - { - "name": "extract_jfk_links", - "categories": ["extract"] - }, - { - "name": "extract_single_link", - "categories": ["extract"] - } - ] -} diff --git a/evals/index.eval.ts b/evals/index.eval.ts deleted file mode 100644 index 617214e1e..000000000 --- a/evals/index.eval.ts +++ /dev/null @@ -1,390 +0,0 @@ -/** - * This script orchestrates the running of evaluations against a set of tasks. - * It uses Braintrust to run multiple testcases (each testcase representing a - * given task-model combination) and then aggregates the results, producing - * a summary of passes, failures, and categorized success rates. - * - * Overview: - * - Reads a configuration file `evals.config.json` to determine what tasks (evaluations) - * are available and which categories they belong to. - * - Supports filtering which tasks to run either by evaluation category or by specific task name. - * - Supports multiple models, defaulting to certain sets of models depending on the category. - * - Runs each selected task against each selected model in parallel, collecting results. - * - Saves a summary of the evaluation results to `eval-summary.json`. - */ -import fs from "fs"; -import path from "path"; -import process from "process"; -import { - DEFAULT_EVAL_CATEGORIES, - filterByCategory, - filterByEvalName, - useTextExtract, -} from "./args"; -import { generateExperimentName } from "./utils"; -import { exactMatch, errorMatch } from "./scoring"; -import { tasksByName, MODELS, tasksConfig } from "./taskConfig"; -import { Eval, wrapAISDKModel, wrapOpenAI } from "braintrust"; -import { EvalFunction, SummaryResult, Testcase } from "@/types/evals"; -import { EvalLogger } from "./logger"; -import { AvailableModel, LLMClient } from "@/dist"; -import { env } from "./env"; -import dotenv from "dotenv"; -import { StagehandEvalError } from "@/types/stagehandErrors"; -import { CustomOpenAIClient } from "@/examples/external_clients/customOpenAI"; -import OpenAI from "openai"; -import { initStagehand } from "./initStagehand"; -import { AISdkClient } from "@/examples/external_clients/aisdk"; -import { google } from "@ai-sdk/google"; -import { anthropic } from "@ai-sdk/anthropic"; -import { groq } from "@ai-sdk/groq"; -import { cerebras } from "@ai-sdk/cerebras"; -dotenv.config(); - -/** - * Read max concurrency and trial count from environment variables set in args.ts. - * Fallback to defaults (20 and 5) if they're not provided. - */ -const MAX_CONCURRENCY = process.env.EVAL_MAX_CONCURRENCY - ? parseInt(process.env.EVAL_MAX_CONCURRENCY, 10) - : 3; - -const TRIAL_COUNT = process.env.EVAL_TRIAL_COUNT - ? parseInt(process.env.EVAL_TRIAL_COUNT, 10) - : 3; - -/** - * generateSummary: - * After all evaluations have finished, aggregate the results into a summary. - * This summary includes: - * - Which tasks passed or failed (with model and categories). - * - Category-wise success percentages. - * - Model-wise success percentages. - * - * The summary is written to `eval-summary.json` for further analysis. - */ -const generateSummary = async ( - results: SummaryResult[], - experimentName: string, -) => { - // Determine passed testcases (those with _success: true) - const passed = results - .filter((r) => r.output._success) - .map((r) => ({ - eval: r.input.name, - model: r.input.modelName, - categories: tasksByName[r.input.name].categories, - })); - - // Determine failed testcases (those with _success: false) - const failed = results - .filter((r) => !r.output._success) - .map((r) => ({ - eval: r.input.name, - model: r.input.modelName, - categories: tasksByName[r.input.name].categories, - })); - - // Calculate success counts for each category - const categorySuccessCounts: Record< - string, - { total: number; success: number } - > = {}; - for (const taskName of Object.keys(tasksByName)) { - const taskCategories = tasksByName[taskName].categories; - const taskResults = results.filter((r) => r.input.name === taskName); - const successCount = taskResults.filter((r) => r.output._success).length; - - for (const cat of taskCategories) { - if (!categorySuccessCounts[cat]) { - categorySuccessCounts[cat] = { total: 0, success: 0 }; - } - categorySuccessCounts[cat].total += taskResults.length; - categorySuccessCounts[cat].success += successCount; - } - } - - // Compute percentage success per category - const categories: Record = {}; - for (const [cat, counts] of Object.entries(categorySuccessCounts)) { - categories[cat] = Math.round((counts.success / counts.total) * 100); - } - - // Compute percentage success per model - const models: Record = {}; - const allModels = [...new Set(results.map((r) => r.input.modelName))]; - for (const model of allModels) { - const modelResults = results.filter((r) => r.input.modelName === model); - const successCount = modelResults.filter((r) => r.output._success).length; - models[model] = Math.round((successCount / modelResults.length) * 100); - } - - // Format and write the summary to a JSON file - const formattedSummary = { - experimentName, - passed, - failed, - categories, - models, - }; - - fs.writeFileSync( - "eval-summary.json", - JSON.stringify(formattedSummary, null, 2), - ); - console.log("Evaluation summary written to eval-summary.json"); -}; - -/** - * generateFilteredTestcases: - * Based on the chosen filters (category or specific eval name) and environment, - * this function generates the set of testcases to run. Each testcase is a combination - * of a task and a model. - * - * Steps: - * - Start with all combinations of tasks (from `tasksByName`) and models (`MODELS`). - * - Filter by category if a category filter was specified. - * - Filter by evaluation name if specified. - * - In the BROWSERBASE environment, exclude certain tasks that are not suitable. - */ -const generateFilteredTestcases = (): Testcase[] => { - // Create a list of all testcases for each model-task combination. - let allTestcases = MODELS.flatMap((model) => - Object.keys(tasksByName).map((testName) => ({ - input: { name: testName, modelName: model }, - name: testName, - tags: [ - model, - testName, - ...(tasksConfig.find((t) => t.name === testName)?.categories || []).map( - (x) => `category/${x}`, - ), - ], - metadata: { - model, - test: testName, - categories: tasksConfig.find((t) => t.name === testName)?.categories, - }, - expected: true, - })), - ); - - // Filter test cases to match default eval categories - allTestcases = allTestcases.filter((testcase) => - DEFAULT_EVAL_CATEGORIES.some((category) => - tasksByName[testcase.name].categories.includes(category), - ), - ); - - // Filter by category if a category is specified - if (filterByCategory) { - allTestcases = allTestcases.filter((testcase) => - tasksByName[testcase.name].categories.includes(filterByCategory!), - ); - } - - // Filter by a specific evaluation (task) name if specified - if (filterByEvalName) { - allTestcases = allTestcases.filter( - (testcase) => - testcase.name === filterByEvalName || - testcase.input.name === filterByEvalName, - ); - } - - // If running in BROWSERBASE environment, exclude tasks that are not applicable. - if (env === "BROWSERBASE") { - allTestcases = allTestcases.filter( - (testcase) => !["peeler_simple", "stock_x"].includes(testcase.name), - ); - } - - console.log( - "All test cases:", - allTestcases - .map( - (t, i) => - `${i}: ${t.name} (${t.input.modelName}): ${t.metadata.categories}`, - ) - .join("\n"), - ); - - return allTestcases; -}; - -/** - * Main execution block: - * - Determine experiment name - * - Determine the project name (braintrustProjectName) based on CI or dev environment - * - Run the Eval function with the given configuration: - * * experimentName: A label for this run - * * data: A function that returns the testcases to run - * * task: A function that executes each task, given input specifying model and task name - * * scores: An array of scoring functions - * * maxConcurrency: Limit on parallel tasks - * * trialCount: Number of trials (retries) per task - * - Collect and summarize results using `generateSummary`. - */ -(async () => { - // Generate a unique name for the experiment - const experimentName: string = generateExperimentName({ - evalName: filterByEvalName || undefined, - category: filterByCategory || undefined, - environment: env, - }); - - // Determine braintrust project name to use (stagehand in CI, stagehand-dev otherwise) - const braintrustProjectName = - process.env.CI === "true" ? "stagehand" : "stagehand-dev"; - - try { - // Run the evaluations with the braintrust Eval function - const evalResult = await Eval(braintrustProjectName, { - experimentName, - data: generateFilteredTestcases, - // Each test is a function that runs the corresponding task module - task: async (input: { name: string; modelName: AvailableModel }) => { - const logger = new EvalLogger(); - try { - // Dynamically import the task based on its name - const taskModulePath = path.join( - __dirname, - "tasks", - `${input.name}.ts`, - ); - const taskModule = (await import(taskModulePath)) as { - [key: string]: EvalFunction; - }; - const taskFunction = taskModule[input.name]; - - if (typeof taskFunction !== "function") { - throw new StagehandEvalError( - `No Eval function found for task name: ${input.name}`, - ); - } - let shouldUseTextExtract = useTextExtract; - const categories = tasksByName[input.name].categories || []; - const isRegression = categories.includes("regression"); - const regressionExtractMethod = tasksByName[input.name].extractMethod; - if (isRegression) { - if (regressionExtractMethod) { - shouldUseTextExtract = regressionExtractMethod === "textExtract"; - } - } - - // Execute the task - let llmClient: LLMClient; - if (input.modelName.startsWith("gpt")) { - llmClient = new CustomOpenAIClient({ - modelName: input.modelName as AvailableModel, - client: wrapOpenAI( - new OpenAI({ - apiKey: process.env.OPENAI_API_KEY, - }), - ), - }); - } else if (input.modelName.startsWith("gemini")) { - llmClient = new AISdkClient({ - model: wrapAISDKModel(google(input.modelName)), - }); - } else if (input.modelName.startsWith("claude")) { - llmClient = new AISdkClient({ - model: wrapAISDKModel(anthropic(input.modelName)), - }); - } else if (input.modelName.includes("groq")) { - llmClient = new AISdkClient({ - model: wrapAISDKModel( - groq( - input.modelName.substring(input.modelName.indexOf("/") + 1), - ), - ), - }); - } else if (input.modelName.includes("cerebras")) { - llmClient = new AISdkClient({ - model: wrapAISDKModel( - cerebras( - input.modelName.substring(input.modelName.indexOf("/") + 1), - ), - ), - }); - } else if (input.modelName.includes("/")) { - llmClient = new CustomOpenAIClient({ - modelName: input.modelName as AvailableModel, - client: wrapOpenAI( - new OpenAI({ - apiKey: process.env.TOGETHER_AI_API_KEY, - baseURL: "https://api.together.xyz/v1", - }), - ), - }); - } - const taskInput = await initStagehand({ - logger, - llmClient, - useTextExtract: shouldUseTextExtract, - }); - let result; - try { - result = await taskFunction(taskInput); - // Log result to console - if (result && result._success) { - console.log(`✅ ${input.name}: Passed`); - } else { - console.log(`❌ ${input.name}: Failed`); - } - } finally { - await taskInput.stagehand.close(); - } - return result; - } catch (error) { - // Log any errors that occur during task execution - console.error(`❌ ${input.name}: Error - ${error}`); - logger.error({ - message: `Error in task ${input.name}`, - level: 0, - auxiliary: { - error: { - value: error.message, - type: "string", - }, - trace: { - value: error.stack, - type: "string", - }, - }, - }); - return { - _success: false, - error: JSON.parse(JSON.stringify(error, null, 2)), - logs: logger.getLogs(), - }; - } - }, - // Use the scoring functions defined above - scores: [exactMatch, errorMatch], - maxConcurrency: MAX_CONCURRENCY, - trialCount: TRIAL_COUNT, - }); - - // Map results to the SummaryResult format - const summaryResults: SummaryResult[] = evalResult.results.map((result) => { - const output = - typeof result.output === "boolean" - ? { _success: result.output } - : result.output; - - return { - input: result.input, - output, - name: result.input.name, - score: output._success ? 1 : 0, - }; - }); - - // Generate and write the summary - await generateSummary(summaryResults, experimentName); - } catch (error) { - console.error("Error during evaluation run:", error); - process.exit(1); - } -})(); diff --git a/evals/initStagehand.ts b/evals/initStagehand.ts deleted file mode 100644 index 174778aea..000000000 --- a/evals/initStagehand.ts +++ /dev/null @@ -1,87 +0,0 @@ -/** - * This file provides a function to initialize a Stagehand instance for use in evaluations. - * It configures the Stagehand environment and sets default options based on the current environment - * (e.g., local or BROWSERBASE), caching preferences, and verbosity. It also establishes a logger for - * capturing logs emitted by Stagehand. - * - * We create a central config object (`StagehandConfig`) that defines all parameters for Stagehand. - * - * The `initStagehand` function takes the model name, an optional DOM settling timeout, and an EvalLogger, - * then uses these to override some default values before creating and initializing the Stagehand instance. - */ - -import { enableCaching, env } from "./env"; -import { ConstructorParams, LLMClient, Stagehand } from "@/dist"; -import { EvalLogger } from "./logger"; -import type { StagehandInitResult } from "@/types/evals"; - -/** - * StagehandConfig: - * This configuration object follows a similar pattern to `examples/stagehand.config.ts`. - * It sets the environment, verbosity, caching preferences, and other defaults. Some values, - * like `apiKey` and `projectId`, can be defined via environment variables if needed. - * - * Adjust or remove fields as appropriate for your environment. - */ -const StagehandConfig = { - env: env, - apiKey: process.env.BROWSERBASE_API_KEY, - projectId: process.env.BROWSERBASE_PROJECT_ID, - verbose: 2 as const, - debugDom: true, - headless: false, - enableCaching, - domSettleTimeoutMs: 30_000, - disablePino: true, -}; - -/** - * Initializes a Stagehand instance for a given model: - * - modelName: The model to use (overrides default in StagehandConfig) - * - domSettleTimeoutMs: Optional timeout for DOM settling operations - * - logger: An EvalLogger instance for capturing logs - * - * Returns: - * - stagehand: The initialized Stagehand instance - * - logger: The provided logger, associated with the Stagehand instance - * - initResponse: Any response data returned by Stagehand initialization - */ -export const initStagehand = async ({ - llmClient, - domSettleTimeoutMs, - logger, - configOverrides, - actTimeoutMs, - useTextExtract, -}: { - llmClient: LLMClient; - domSettleTimeoutMs?: number; - logger: EvalLogger; - configOverrides?: Partial; - actTimeoutMs?: number; - useTextExtract?: boolean; -}): Promise => { - const config = { - ...StagehandConfig, - llmClient, - ...(domSettleTimeoutMs && { domSettleTimeoutMs }), - actTimeoutMs, - ...configOverrides, - logger: logger.log.bind(logger), - }; - - const stagehand = new Stagehand(config); - - // Associate the logger with the Stagehand instance - logger.init(stagehand); - - const { debugUrl, sessionUrl } = await stagehand.init(); - return { - stagehand, - stagehandConfig: config, - logger, - debugUrl, - sessionUrl, - useTextExtract, - }; -}; diff --git a/evals/llm_clients/hn_aisdk.ts b/evals/llm_clients/hn_aisdk.ts deleted file mode 100644 index 51ab4937e..000000000 --- a/evals/llm_clients/hn_aisdk.ts +++ /dev/null @@ -1,119 +0,0 @@ -import { Stagehand } from "@/dist"; -import { AISdkClient } from "@/examples/external_clients/aisdk"; -import { EvalFunction } from "@/types/evals"; -import { openai } from "@ai-sdk/openai/dist"; -import { z } from "zod"; - -export const hn_aisdk: EvalFunction = async ({ - debugUrl, - sessionUrl, - stagehandConfig, - logger, -}) => { - const stagehand = new Stagehand({ - ...stagehandConfig, - llmClient: new AISdkClient({ - model: openai("gpt-4o-mini"), - }), - }); - await stagehand.init(); - await stagehand.page.goto( - "https://browserbase.github.io/stagehand-eval-sites/sites/hackernews/", - ); - - let { story } = await stagehand.page.extract({ - instruction: "extract the title of the top story on the page", - schema: z.object({ - story: z.string().describe("the title of the top story on the page"), - }), - }); - // remove the (url) part of the story title - story = story.split(" (")[0]; - - const expectedStoryElement = await stagehand.page.$( - "xpath=/html/body/center/table/tbody/tr[3]/td/table/tbody/tr[1]/td[3]/span/a", - ); - // remove the (url) part of the story title - const expectedStory = (await expectedStoryElement?.textContent())?.split( - " (", - )?.[0]; - - if (!expectedStory) { - logger.error({ - message: "Could not find expected story element", - level: 0, - }); - return { - _success: false, - error: "Could not find expected story element", - debugUrl, - sessionUrl, - logs: logger.getLogs(), - }; - } - - if (story !== expectedStory) { - logger.error({ - message: "Extracted story does not match expected story", - level: 0, - auxiliary: { - expected: { - value: expectedStory, - type: "string", - }, - actual: { - value: story, - type: "string", - }, - }, - }); - return { - _success: false, - error: "Extracted story does not match expected story", - expectedStory, - actualStory: story, - debugUrl, - sessionUrl, - logs: logger.getLogs(), - }; - } - - await stagehand.page.act("Click on the 'new' tab"); - - if (stagehand.page.url() !== "https://news.ycombinator.com/newest") { - logger.error({ - message: "Page did not navigate to the 'new' tab", - level: 0, - auxiliary: { - expected: { - value: "https://news.ycombinator.com/newest", - type: "string", - }, - actual: { - value: stagehand.page.url(), - type: "string", - }, - }, - }); - return { - _success: false, - error: "Page did not navigate to the 'new' tab", - expectedUrl: "https://news.ycombinator.com/newest", - actualUrl: stagehand.page.url(), - debugUrl, - sessionUrl, - logs: logger.getLogs(), - }; - } - - await stagehand.close(); - - return { - _success: true, - expectedStory, - actualStory: story, - debugUrl, - sessionUrl, - logs: logger.getLogs(), - }; -}; diff --git a/evals/llm_clients/hn_customOpenAI.ts b/evals/llm_clients/hn_customOpenAI.ts deleted file mode 100644 index 7b4f1f1e5..000000000 --- a/evals/llm_clients/hn_customOpenAI.ts +++ /dev/null @@ -1,124 +0,0 @@ -import { EvalFunction } from "@/types/evals"; -import { z } from "zod"; -import { CustomOpenAIClient } from "@/examples/external_clients/customOpenAI"; -import OpenAI from "openai"; -import { Stagehand } from "@/dist"; - -export const hn_customOpenAI: EvalFunction = async ({ - logger, - stagehandConfig, - debugUrl, - sessionUrl, -}) => { - const stagehand = new Stagehand({ - ...stagehandConfig, - llmClient: new CustomOpenAIClient({ - modelName: "gpt-4o-mini", - client: new OpenAI({ - apiKey: process.env.OPENAI_API_KEY, - }), - }), - }); - - await stagehand.init(); - - await stagehand.page.goto( - "https://browserbase.github.io/stagehand-eval-sites/sites/hackernews/", - ); - - let { story } = await stagehand.page.extract({ - instruction: "extract the title of the top story on the page", - schema: z.object({ - story: z.string().describe("the title of the top story on the page"), - }), - }); - // remove the (url) part of the story title - story = story.split(" (")[0]; - - const expectedStoryElement = await stagehand.page.$( - "xpath=/html/body/center/table/tbody/tr[3]/td/table/tbody/tr[1]/td[3]/span/a", - ); - // remove the (url) part of the story title - const expectedStory = (await expectedStoryElement?.textContent())?.split( - " (", - )?.[0]; - - if (!expectedStory) { - logger.error({ - message: "Could not find expected story element", - level: 0, - }); - return { - _success: false, - error: "Could not find expected story element", - debugUrl, - sessionUrl, - logs: logger.getLogs(), - }; - } - - if (story !== expectedStory) { - logger.error({ - message: "Extracted story does not match expected story", - level: 0, - auxiliary: { - expected: { - value: expectedStory, - type: "string", - }, - actual: { - value: story, - type: "string", - }, - }, - }); - return { - _success: false, - error: "Extracted story does not match expected story", - expectedStory, - actualStory: story, - debugUrl, - sessionUrl, - logs: logger.getLogs(), - }; - } - - await stagehand.page.act("Click on the 'new' tab"); - - if (stagehand.page.url() !== "https://news.ycombinator.com/newest") { - logger.error({ - message: "Page did not navigate to the 'new' tab", - level: 0, - auxiliary: { - expected: { - value: "https://news.ycombinator.com/newest", - type: "string", - }, - actual: { - value: stagehand.page.url(), - type: "string", - }, - }, - }); - return { - _success: false, - error: "Page did not navigate to the 'new' tab", - expectedUrl: "https://news.ycombinator.com/newest", - actualUrl: stagehand.page.url(), - debugUrl, - sessionUrl, - logs: logger.getLogs(), - }; - } - - await stagehand.close(); - - return { - _success: true, - expectedStory, - actualStory: story, - debugUrl, - sessionUrl, - logs: logger.getLogs(), - }; -}; diff --git a/evals/llm_clients/hn_langchain.ts b/evals/llm_clients/hn_langchain.ts deleted file mode 100644 index 35fc6b68d..000000000 --- a/evals/llm_clients/hn_langchain.ts +++ /dev/null @@ -1,122 +0,0 @@ -import { EvalFunction } from "@/types/evals"; -import { z } from "zod"; -import { LangchainClient } from "@/examples/external_clients/langchain"; -import { ChatOpenAI } from "@langchain/openai"; -import { Stagehand } from "@/dist"; - -export const hn_langchain: EvalFunction = async ({ - logger, - stagehandConfig, - debugUrl, - sessionUrl, -}) => { - const stagehand = new Stagehand({ - ...stagehandConfig, - llmClient: new LangchainClient( - new ChatOpenAI({ - model: "gpt-4o-mini", - }), - ), - }); - await stagehand.init(); - - await stagehand.page.goto( - "https://browserbase.github.io/stagehand-eval-sites/sites/hackernews/", - ); - - let { story } = await stagehand.page.extract({ - instruction: "extract the title of the top story on the page", - schema: z.object({ - story: z.string().describe("the title of the top story on the page"), - }), - }); - // remove the (url) part of the story title - story = story.split(" (")[0]; - - const expectedStoryElement = await stagehand.page.$( - "xpath=/html/body/center/table/tbody/tr[3]/td/table/tbody/tr[1]/td[3]/span/a", - ); - // remove the (url) part of the story title - const expectedStory = (await expectedStoryElement?.textContent())?.split( - " (", - )?.[0]; - - if (!expectedStory) { - logger.error({ - message: "Could not find expected story element", - level: 0, - }); - return { - _success: false, - error: "Could not find expected story element", - debugUrl, - sessionUrl, - logs: logger.getLogs(), - }; - } - - if (story !== expectedStory) { - logger.error({ - message: "Extracted story does not match expected story", - level: 0, - auxiliary: { - expected: { - value: expectedStory, - type: "string", - }, - actual: { - value: story, - type: "string", - }, - }, - }); - return { - _success: false, - error: "Extracted story does not match expected story", - expectedStory, - actualStory: story, - debugUrl, - sessionUrl, - logs: logger.getLogs(), - }; - } - - await stagehand.page.act("Click on the 'new' tab"); - - if (stagehand.page.url() !== "https://news.ycombinator.com/newest") { - logger.error({ - message: "Page did not navigate to the 'new' tab", - level: 0, - auxiliary: { - expected: { - value: "https://news.ycombinator.com/newest", - type: "string", - }, - actual: { - value: stagehand.page.url(), - type: "string", - }, - }, - }); - return { - _success: false, - error: "Page did not navigate to the 'new' tab", - expectedUrl: "https://news.ycombinator.com/newest", - actualUrl: stagehand.page.url(), - debugUrl, - sessionUrl, - logs: logger.getLogs(), - }; - } - - await stagehand.close(); - - return { - _success: true, - expectedStory, - actualStory: story, - debugUrl, - sessionUrl, - logs: logger.getLogs(), - }; -}; diff --git a/evals/logger.ts b/evals/logger.ts deleted file mode 100644 index c234ada44..000000000 --- a/evals/logger.ts +++ /dev/null @@ -1,123 +0,0 @@ -/** - * This file defines the `EvalLogger` class, which is used to capture and manage - * log lines during the evaluation process. The logger supports different log - * levels (info, error, warn), stores logs in memory for later retrieval, and - * also prints them to the console for immediate feedback. - * - * The `parseLogLine` function helps transform raw `LogLine` objects into a more - * structured format (`LogLineEval`), making auxiliary data easier to understand - * and analyze. By associating an `EvalLogger` instance with a `Stagehand` object, - * all logs emitted during the evaluation process can be captured, persisted, and - * reviewed after the tasks complete. - */ -import { logLineToString } from "./utils"; -import { LogLineEval } from "@/types/evals"; -import { Stagehand, LogLine } from "@/dist"; - -/** - * parseLogLine: - * Given a LogLine, attempts to parse its `auxiliary` field into a structured object. - * If parsing fails, logs an error and returns the original line. - * - * The `auxiliary` field in the log line typically contains additional metadata about the log event. - */ -function parseLogLine(logLine: LogLine): LogLineEval { - try { - let parsedAuxiliary: Record | undefined; - - if (logLine.auxiliary) { - parsedAuxiliary = {}; - - for (const [key, entry] of Object.entries(logLine.auxiliary)) { - try { - parsedAuxiliary[key] = - entry.type === "object" ? JSON.parse(entry.value) : entry.value; - } catch (parseError) { - console.warn(`Failed to parse auxiliary entry ${key}:`, parseError); - // If parsing fails, use the raw value - parsedAuxiliary[key] = entry.value; - } - } - } - - return { - ...logLine, - auxiliary: undefined, - parsedAuxiliary, - } as LogLineEval; - } catch (e) { - console.log("Error parsing log line", logLine); - console.error(e); - return logLine; - } -} - -/** - * EvalLogger: - * A logger class used during evaluations to capture and print log lines. - * - * Capabilities: - * - Maintains an internal array of log lines (EvalLogger.logs) for later retrieval. - * - Can be initialized with a Stagehand instance to provide consistent logging. - * - Supports logging at different levels (info, error, warn). - * - Each log line is converted to a string and printed to console for immediate feedback. - * - Also keeps a structured version of the logs that can be returned for analysis or - * included in evaluation output. - */ -export class EvalLogger { - private logs: LogLineEval[] = []; - stagehand?: Stagehand; - - constructor() { - this.logs = []; - } - - /** - * init: - * Associates this logger with a given Stagehand instance. - * This allows the logger to provide additional context if needed. - */ - init(stagehand: Stagehand) { - this.stagehand = stagehand; - } - - /** - * log: - * Logs a message at the default (info) level. - * Uses `logLineToString` to produce a readable output on the console, - * and then stores the parsed log line in `this.logs`. - */ - log(logLine: LogLine) { - console.log(logLineToString(logLine)); - this.logs.push(parseLogLine(logLine)); - } - - /** - * error: - * Logs an error message with `console.error` and stores it. - * Useful for capturing and differentiating error-level logs. - */ - error(logLine: LogLine) { - console.error(logLineToString(logLine)); - this.logs.push(parseLogLine(logLine)); - } - - /** - * warn: - * Logs a warning message with `console.warn` and stores it. - * Helps differentiate warnings from regular info logs. - */ - warn(logLine: LogLine) { - console.warn(logLineToString(logLine)); - this.logs.push(parseLogLine(logLine)); - } - - /** - * getLogs: - * Retrieves the array of stored log lines. - * Useful for returning logs after a task completes, for analysis or debugging. - */ - getLogs(): LogLineEval[] { - return this.logs || []; - } -} diff --git a/evals/scoring.ts b/evals/scoring.ts deleted file mode 100644 index 42552e5e1..000000000 --- a/evals/scoring.ts +++ /dev/null @@ -1,65 +0,0 @@ -/** - * This file implements scoring functions needed by braintrust. - */ - -import { EvalArgs, EvalInput, EvalResult } from "@/types/evals"; - -/** - * Scoring function: exactMatch - * Given the arguments (including input, output, and expected result), - * this returns a score of 1 if the result matches the expectation, and 0 otherwise. - * - * If "expected" is true, it checks if the output indicates success. - * If "expected" is a boolean or an object with _success flag, - * it checks if output is exactly that success condition. - */ -export function exactMatch( - args: EvalArgs, -): EvalResult { - console.log(`Task "${args.input.name}" returned: ${args.output}`); - - const expected = args.expected ?? true; - if (expected === true) { - // If we expect a success (true), then we check the output's _success flag. - return { - name: "Exact match", - score: - typeof args.output === "boolean" - ? args.output - ? 1 - : 0 - : args.output._success - ? 1 - : 0, - }; - } - - // If expected is not true, just directly compare the output to expected. - return { - name: "Exact match", - score: args.output === expected ? 1 : 0, - }; -} - -/** - * Scoring function: errorMatch - * Determines if an error occurred in the task. - * Scores 1 if an error is found, otherwise 0. - */ -export function errorMatch( - args: EvalArgs< - EvalInput, - boolean | { _success: boolean; error?: unknown }, - unknown - >, -): EvalResult { - console.log(`Task "${args.input.name}" returned: ${args.output}`); - - return { - name: "Error rate", - score: - typeof args.output === "object" && args.output.error !== undefined - ? 1 - : 0, - }; -} diff --git a/evals/taskConfig.ts b/evals/taskConfig.ts deleted file mode 100644 index e5ea48b02..000000000 --- a/evals/taskConfig.ts +++ /dev/null @@ -1,140 +0,0 @@ -/** - * This file is responsible for: - * - Loading and parsing the `evals.config.json` file, which defines tasks (evaluations) and their associated categories. - * - Building a lookup structure (`tasksByName`) to map each task name to its categories. - * - Filtering tasks based on command-line arguments (e.g., `filterByEvalName`) and ensuring that requested tasks exist. - * - Determining which models to use for evaluations, depending on the category and environment variables. - * - Validating that the chosen models are supported. - * - * The exported objects (`tasksByName`, `MODELS`, `config`) are used by the main evaluation script and other modules - * to know which tasks and models are available, and to configure the evaluations accordingly. - */ - -import fs from "fs"; -import path from "path"; -import { AvailableModel } from "@/dist"; -import { filterByEvalName } from "./args"; - -const ALL_EVAL_MODELS = [ - // GOOGLE - "gemini-2.0-flash", - "gemini-2.0-flash-lite", - "gemini-1.5-flash", - "gemini-2.5-pro-exp-03-25", - "gemini-1.5-pro", - "gemini-1.5-flash-8b", - // ANTHROPIC - "claude-3-5-sonnet-latest", - "claude-3-7-sonnet-latest", - // OPENAI - "gpt-4o-mini", - "gpt-4o", - "gpt-4.5-preview", - // TOGETHER - META - "meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo", - "meta-llama/Llama-3.3-70B-Instruct-Turbo", - "meta-llama/Llama-4-Scout-17B-16E-Instruct", - "meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8", - // TOGETHER - DEEPSEEK - "deepseek-ai/DeepSeek-V3", - "Qwen/Qwen2.5-7B-Instruct-Turbo", - // GROQ - "groq/meta-llama/llama-4-scout-17b-16e-instruct", - "groq/llama-3.3-70b-versatile", - "groq/llama3-70b-8192", - "groq/qwen-qwq-32b", - "groq/qwen-2.5-32b", - "groq/deepseek-r1-distill-qwen-32b", - "groq/deepseek-r1-distill-llama-70b", - // CEREBRAS - "cerebras/llama3.3-70b", -]; - -// The configuration file `evals.config.json` contains a list of tasks and their associated categories. -const configPath = path.join(__dirname, "evals.config.json"); -const config = JSON.parse(fs.readFileSync(configPath, "utf-8")) satisfies { - tasks: { - name: string; - categories: string[]; - }[]; -}; - -/** - * The `tasksConfig` defines all tasks from the config file. Each task has a name and categories. - * We create a mapping `tasksByName` from task name to its categories for quick lookup. - */ -type TaskConfig = { - name: string; - categories: string[]; - extract_method?: string; -}; -const tasksConfig = config.tasks as TaskConfig[]; - -const tasksByName = tasksConfig.reduce< - Record ->((acc, task) => { - acc[task.name] = { - categories: task.categories, - extractMethod: task.extract_method, - }; - return acc; -}, {}); - -/** - * If filtering by a specific eval name (task), ensure that this task actually exists. - */ -if (filterByEvalName && !tasksByName[filterByEvalName]) { - console.error(`Error: Evaluation "${filterByEvalName}" does not exist.`); - process.exit(1); -} - -/** - * Determine which models to run the evaluations against. - * - * DEFAULT_EVAL_MODELS: The default set of models used for most categories. - */ -const DEFAULT_EVAL_MODELS = process.env.EVAL_MODELS - ? process.env.EVAL_MODELS.split(",") - : ["claude-3-5-sonnet-latest", "gpt-4o-mini", "gpt-4o"]; - -/** - * getModelList: - * Returns a list of models to be used for the given category. - * If category is "experimental", it merges DEFAULT_EVAL_MODELS and EXPERIMENTAL_EVAL_MODELS. - * Otherwise, returns DEFAULT_EVAL_MODELS filtered by provider if specified. - */ -const getModelList = (): string[] => { - const provider = process.env.EVAL_PROVIDER?.toLowerCase(); - - if (!provider) { - return DEFAULT_EVAL_MODELS; - } - - return ALL_EVAL_MODELS.filter((model) => { - const modelLower = model.toLowerCase(); - if (provider === "openai") { - return modelLower.startsWith("gpt"); - } else if (provider === "anthropic") { - return modelLower.startsWith("claude"); - } else if (provider === "google") { - return modelLower.startsWith("gemini"); - } else if (provider === "together") { - return ( - modelLower.startsWith("meta-llama") || - modelLower.startsWith("llama") || - modelLower.startsWith("deepseek") || - modelLower.startsWith("qwen") - ); - } else if (provider === "groq") { - return modelLower.startsWith("groq"); - } else if (provider === "cerebras") { - return modelLower.startsWith("cerebras"); - } - return true; - }); -}; -const MODELS: AvailableModel[] = getModelList().map((model) => { - return model as AvailableModel; -}); - -export { tasksByName, MODELS, tasksConfig }; diff --git a/evals/tasks/allrecipes.ts b/evals/tasks/allrecipes.ts deleted file mode 100644 index e1b8a0c50..000000000 --- a/evals/tasks/allrecipes.ts +++ /dev/null @@ -1,92 +0,0 @@ -import { EvalFunction } from "@/types/evals"; -import { z } from "zod"; - -export const allrecipes: EvalFunction = async ({ - logger, - useTextExtract, - debugUrl, - sessionUrl, - stagehand, -}) => { - await stagehand.page.goto("https://www.allrecipes.com/", { - waitUntil: "domcontentloaded", - }); - - await stagehand.page.act({ - action: 'Type "chocolate chip cookies" in the search bar', - }); - await stagehand.page.act({ - action: "press enter", - }); - - const recipeDetails = await stagehand.page.extract({ - instruction: - "Extract the title of the first recipe and the total number of ratings it has received.", - schema: z.object({ - title: z.string().describe("Title of the recipe"), - total_ratings: z - .string() - .describe("Total number of ratings for the recipe"), - }), - useTextExtract, - }); - - await stagehand.close(); - - const { title, total_ratings } = recipeDetails; - const expectedTitle = "Best Chocolate Chip Cookies"; - const expectedRatings = 19164; - - const extractedRatings = parseInt(total_ratings.replace(/[^\d]/g, ""), 10); - const isRatingsWithinRange = - extractedRatings >= expectedRatings - 1000 && - extractedRatings <= expectedRatings + 1000; - - if (title !== expectedTitle || !isRatingsWithinRange) { - const errors = []; - if (title !== expectedTitle) { - errors.push({ - message: "Extracted title does not match the expected title", - expected: expectedTitle, - actual: title, - }); - } - if (!isRatingsWithinRange) { - errors.push({ - message: "Extracted ratings are not within the expected range", - expected: `${expectedRatings} ± 1000`, - actual: extractedRatings.toString(), - }); - } - - logger.error({ - message: "Failed to extract correct recipe details", - level: 0, - auxiliary: { - errors: { - value: JSON.stringify(errors), - type: "object", - }, - }, - }); - - return { - _success: false, - error: "Recipe details extraction validation failed", - logs: logger.getLogs(), - debugUrl, - sessionUrl, - }; - } - - return { - _success: true, - recipeDetails: { - title, - total_ratings: extractedRatings, - }, - logs: logger.getLogs(), - debugUrl, - sessionUrl, - }; -}; diff --git a/evals/tasks/amazon_add_to_cart.ts b/evals/tasks/amazon_add_to_cart.ts deleted file mode 100644 index 64c410f8f..000000000 --- a/evals/tasks/amazon_add_to_cart.ts +++ /dev/null @@ -1,39 +0,0 @@ -import { EvalFunction } from "@/types/evals"; - -export const amazon_add_to_cart: EvalFunction = async ({ - logger, - debugUrl, - sessionUrl, - stagehand, -}) => { - await stagehand.page.goto( - "https://browserbase.github.io/stagehand-eval-sites/sites/amazon/", - ); - - await stagehand.page.waitForTimeout(5000); - - await stagehand.page.act({ - action: "click the 'Add to Cart' button", - }); - - await stagehand.page.waitForTimeout(2000); - - await stagehand.page.act({ - action: "click the 'Proceed to checkout' button", - }); - - await stagehand.page.waitForTimeout(2000); - const currentUrl = stagehand.page.url(); - const expectedUrl = - "https://browserbase.github.io/stagehand-eval-sites/sites/amazon/sign-in.html"; - - await stagehand.close(); - - return { - _success: currentUrl === expectedUrl, - currentUrl, - debugUrl, - sessionUrl, - logs: logger.getLogs(), - }; -}; diff --git a/evals/tasks/apple.ts b/evals/tasks/apple.ts deleted file mode 100644 index 1926a1718..000000000 --- a/evals/tasks/apple.ts +++ /dev/null @@ -1,39 +0,0 @@ -import { EvalFunction } from "@/types/evals"; - -export const apple: EvalFunction = async ({ - logger, - debugUrl, - sessionUrl, - stagehand, -}) => { - await stagehand.page.goto("https://www.apple.com/iphone-16-pro/"); - - await stagehand.page.act({ action: "click on the buy button" }); - await stagehand.page.act({ action: "select the Pro Max model" }); - await stagehand.page.act({ action: "select the natural titanium color" }); - await stagehand.page.act({ action: "select the 256GB storage option" }); - await stagehand.page.act({ - action: "click on the 'select a smartphone' trade-in option", - }); - - await stagehand.page.act({ - action: "select the iPhone 13 mini model from the dropdown", - }); - await stagehand.page.act({ - action: "select the iPhone 13 mini is in good condition", - }); - - const successMessageLocator = stagehand.page.locator( - 'text="Good News. Your iPhone 13 mini qualifies for credit."', - ); - const isVisible = await successMessageLocator.isVisible(); - - await stagehand.close(); - - return { - _success: isVisible, - debugUrl, - sessionUrl, - logs: logger.getLogs(), - }; -}; diff --git a/evals/tasks/arxiv.ts b/evals/tasks/arxiv.ts deleted file mode 100644 index 919d28c7f..000000000 --- a/evals/tasks/arxiv.ts +++ /dev/null @@ -1,198 +0,0 @@ -import { EvalFunction } from "@/types/evals"; -import { z } from "zod"; - -export const arxiv: EvalFunction = async ({ - logger, - debugUrl, - sessionUrl, - stagehand, - useTextExtract, -}) => { - try { - await stagehand.page.goto("https://arxiv.org/search/"); - - await stagehand.page.act( - "type web agents with multimodal models in the search bar", - ); - - await stagehand.page.act("hit enter"); - - const paper_links = await stagehand.page.extract({ - instruction: "extract the titles and links for two papers", - schema: z.object({ - papers: z - .array( - z.object({ - title: z.string().describe("the title of the paper"), - link: z.string().describe("the link to the paper").nullable(), - }), - ) - .describe("list of papers"), - }), - useTextExtract, - }); - - if ( - !paper_links || - !paper_links.papers || - paper_links.papers.length === 0 - ) { - await stagehand.close(); - - return { - _success: false, - logs: logger.getLogs(), - debugUrl, - sessionUrl, - }; - } - - const papers = []; - for (const paper of paper_links.papers) { - if (paper.link) { - await stagehand.page.goto(paper.link); - const abstract = await stagehand.page.extract({ - instruction: "extract details of the paper from the abstract", - schema: z.object({ - category: z - .string() - .describe( - "the category of the paper. one of {'Benchmark', 'Dataset', 'Model', 'Framework', 'System', 'Other'}", - ), - problem: z - .string() - .describe( - "summarize the problem that the paper is trying to solve in one sentence", - ) - .nullable(), - methodology: z - .string() - .describe( - "summarize the methodology of the paper in one sentence", - ) - .nullable(), - results: z - .string() - .describe("summarize the results of the paper in one sentence") - .nullable(), - conclusion: z - .string() - .describe("summarize the conclusion of the paper in one sentence") - .nullable(), - code: z - .string() - .describe( - "if provided, extract only the link to the code repository, without additional text. this is often optional and not always provided.", - ) - .nullable(), - }), - useTextExtract, - }); - - papers.push({ - title: paper.title, - link: paper.link, - ...abstract, - }); - } - } - - if (!papers || papers.length === 0) { - await stagehand.close(); - - return { - _success: false, - logs: logger.getLogs(), - debugUrl, - sessionUrl, - }; - } - - if (papers.length !== 2) { - logger.error({ - message: "incorrect number of papers extracted", - level: 0, - auxiliary: { - expected: { - value: "2", - type: "integer", - }, - actual: { - value: papers.length.toString(), - type: "integer", - }, - }, - }); - - await stagehand.close(); - - return { - _success: false, - error: "Incorrect number of papers extracted", - logs: logger.getLogs(), - debugUrl, - sessionUrl, - }; - } - - // Ensure that every paper has a problem and methodology - for (const paper of papers) { - if (!paper.problem || !paper.methodology) { - logger.error({ - message: `paper missing problem or methodology`, - level: 0, - auxiliary: { - paper: { - value: JSON.stringify(paper), - type: "object", - }, - }, - }); - - await stagehand.close(); - - return { - _success: false, - error: "Incomplete paper information", - logs: logger.getLogs(), - debugUrl, - sessionUrl, - }; - } - } - - await stagehand.close(); - - return { - _success: true, - papers, - logs: logger.getLogs(), - debugUrl, - sessionUrl, - }; - } catch (error) { - logger.error({ - message: `error in arxiv function`, - level: 0, - auxiliary: { - error: { - value: error.message, - type: "string", - }, - trace: { - value: error.stack, - type: "string", - }, - }, - }); - - await stagehand.close(); - - return { - _success: false, - logs: logger.getLogs(), - debugUrl, - sessionUrl, - }; - } -}; diff --git a/evals/tasks/bidnet.ts b/evals/tasks/bidnet.ts deleted file mode 100644 index d73c83030..000000000 --- a/evals/tasks/bidnet.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { EvalFunction } from "@/types/evals"; - -export const bidnet: EvalFunction = async ({ - logger, - debugUrl, - sessionUrl, - stagehand, -}) => { - await stagehand.page.goto("https://www.bidnetdirect.com/"); - - await stagehand.page.act({ - action: 'Click on the "Construction" keyword', - }); - - const expectedUrl = - "https://www.bidnetdirect.com/public/solicitations/open?keywords=Construction"; - const currentUrl = stagehand.page.url(); - - await stagehand.close(); - - return { - _success: currentUrl.startsWith(expectedUrl), - currentUrl, - debugUrl, - sessionUrl, - logs: logger.getLogs(), - }; -}; diff --git a/evals/tasks/combination_sauce.ts b/evals/tasks/combination_sauce.ts deleted file mode 100644 index fc8bfc32e..000000000 --- a/evals/tasks/combination_sauce.ts +++ /dev/null @@ -1,69 +0,0 @@ -import { EvalFunction } from "@/types/evals"; -import { z } from "zod"; - -export const combination_sauce: EvalFunction = async ({ - logger, - useTextExtract, - debugUrl, - sessionUrl, - stagehand, -}) => { - try { - await stagehand.page.goto("https://www.saucedemo.com/"); - - const { usernames, password } = await stagehand.page.extract({ - instruction: "extract the accepted usernames and the password for login", - schema: z.object({ - usernames: z.array(z.string()).describe("the accepted usernames"), - password: z.string().describe("the password for login"), - }), - useTextExtract, - }); - - await stagehand.page.act({ - action: `enter username 'standard_user'`, - }); - - await stagehand.page.act({ - action: `enter password '${password}'`, - }); - - await stagehand.page.act({ - action: "click on 'login'", - }); - - const observations = await stagehand.page.observe({ - instruction: "find all the 'add to cart' buttons", - }); - - console.log("observations", observations); - console.log("observations length", observations.length); - - const url = await stagehand.page.url(); - - await stagehand.close(); - - const usernamesCheck = usernames.length === 6; - const urlCheck = url === "https://www.saucedemo.com/inventory.html"; - const observationsCheck = observations.length === 6; - - return { - _success: usernamesCheck && urlCheck && observationsCheck, - debugUrl, - sessionUrl, - logs: logger.getLogs(), - }; - } catch (error) { - console.error("Error or timeout occurred:", error); - - await stagehand.close(); - - return { - _success: false, - error: JSON.parse(JSON.stringify(error, null, 2)), - debugUrl, - sessionUrl, - logs: logger.getLogs(), - }; - } -}; diff --git a/evals/tasks/costar.ts b/evals/tasks/costar.ts deleted file mode 100644 index 221919e4e..000000000 --- a/evals/tasks/costar.ts +++ /dev/null @@ -1,84 +0,0 @@ -import { EvalFunction } from "@/types/evals"; -import { z } from "zod"; - -export const costar: EvalFunction = async ({ - logger, - debugUrl, - sessionUrl, - stagehand, - useTextExtract, -}) => { - // TODO: fix this eval - does not work in headless mode - try { - await Promise.race([ - stagehand.page.goto("https://www.costar.com/"), - new Promise((_, reject) => - setTimeout(() => reject(new Error("Navigation timeout")), 30000), - ), - ]); - - await stagehand.page.act({ action: "click on the first article" }); - - await stagehand.page.act({ - action: "click on the learn more button for the first job", - }); - - const articleTitle = await stagehand.page.extract({ - instruction: "extract the title of the article", - schema: z.object({ - title: z.string().describe("the title of the article").nullable(), - }), - useTextExtract, - }); - - logger.log({ - message: "got article title", - level: 1, - auxiliary: { - articleTitle: { - value: JSON.stringify(articleTitle), - type: "object", - }, - }, - }); - - // Check if the title is more than 5 characters - const isTitleValid = - articleTitle.title !== null && articleTitle.title.length > 5; - - await stagehand.close(); - - return { - title: articleTitle.title, - _success: isTitleValid, - debugUrl, - sessionUrl, - logs: logger.getLogs(), - }; - } catch (error) { - logger.error({ - message: "error in costar function", - level: 0, - auxiliary: { - error: { - value: error.message, - type: "string", - }, - trace: { - value: error.stack, - type: "string", - }, - }, - }); - - await stagehand.close(); - - return { - title: null, - _success: false, - debugUrl, - sessionUrl, - logs: logger.getLogs(), - }; - } -}; diff --git a/evals/tasks/expect_act_timeout.ts b/evals/tasks/expect_act_timeout.ts deleted file mode 100644 index 0a6f45170..000000000 --- a/evals/tasks/expect_act_timeout.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { EvalFunction } from "@/types/evals"; - -export const expect_act_timeout: EvalFunction = async ({ - logger, - debugUrl, - sessionUrl, - stagehand, -}) => { - await stagehand.page.goto("https://docs.stagehand.dev"); - const result = await stagehand.page.act({ - action: "search for 'Stagehand'", - timeoutMs: 1_000, - }); - - await stagehand.close(); - - return { - _success: !result.success, - debugUrl, - sessionUrl, - logs: logger.getLogs(), - }; -}; diff --git a/evals/tasks/expedia.ts b/evals/tasks/expedia.ts deleted file mode 100644 index ac5645a5d..000000000 --- a/evals/tasks/expedia.ts +++ /dev/null @@ -1,45 +0,0 @@ -import { EvalFunction } from "@/types/evals"; - -export const expedia: EvalFunction = async ({ - logger, - debugUrl, - sessionUrl, - stagehand, -}) => { - try { - await stagehand.page.goto("https://www.expedia.com/flights"); - await stagehand.page.act( - "find round-trip flights from San Francisco (SFO) to Toronto (YYZ) for Jan 1, 2025 (up to one to two weeks)", - ); - await stagehand.page.act("Go to the first non-stop flight"); - await stagehand.page.act("select the cheapest flight"); - await stagehand.page.act("click on the first non-stop flight"); - await stagehand.page.act("Take me to the checkout page"); - - const url = stagehand.page.url(); - return { - _success: url.startsWith("https://www.expedia.com/Checkout/"), - logs: logger.getLogs(), - debugUrl, - sessionUrl, - }; - } catch (error) { - logger.error({ - message: "Error in expedia eval", - level: 0, - auxiliary: { - error: { value: error.message, type: "string" }, - trace: { value: error.stack, type: "string" }, - }, - }); - - return { - _success: false, - logs: logger.getLogs(), - debugUrl, - sessionUrl, - }; - } finally { - await stagehand.close(); - } -}; diff --git a/evals/tasks/expedia_search.ts b/evals/tasks/expedia_search.ts deleted file mode 100644 index 592e553dd..000000000 --- a/evals/tasks/expedia_search.ts +++ /dev/null @@ -1,58 +0,0 @@ -import { EvalFunction } from "@/types/evals"; - -export const expedia_search: EvalFunction = async ({ - logger, - debugUrl, - sessionUrl, - stagehand, -}) => { - try { - await stagehand.page.goto("https://www.expedia.com/flights"); - await stagehand.page.act({ - action: - "find round-trip flights from San Francisco (SFO) to Toronto (YYZ) for Jan 1, 2025 (up to one to two weeks)", - }); - - await stagehand.page.act({ action: "Go to the first non-stop flight" }); - - await stagehand.page.act({ action: "select the cheapest flight" }); - - await stagehand.page.act({ action: "click on the first non-stop flight" }); - - await stagehand.page.act({ - action: "Take me to the checkout page", - }); - - const url = stagehand.page.url(); - return { - _success: url.startsWith("https://www.expedia.com/Checkout/"), - logs: logger.getLogs(), - debugUrl, - sessionUrl, - }; - } catch (error) { - logger.error({ - message: `error in expedia function`, - level: 0, - auxiliary: { - error: { - value: JSON.stringify(error, null, 2), - type: "object", - }, - trace: { - value: error.stack, - type: "string", - }, - }, - }); - return { - _success: false, - error: JSON.parse(JSON.stringify(error, null, 2)), - debugUrl, - sessionUrl, - logs: logger.getLogs(), - }; - } finally { - await stagehand.close(); - } -}; diff --git a/evals/tasks/extract_aigrant_companies.ts b/evals/tasks/extract_aigrant_companies.ts deleted file mode 100644 index f3e1f19cb..000000000 --- a/evals/tasks/extract_aigrant_companies.ts +++ /dev/null @@ -1,127 +0,0 @@ -import { z } from "zod"; -import { EvalFunction } from "@/types/evals"; - -export const extract_aigrant_companies: EvalFunction = async ({ - logger, - debugUrl, - sessionUrl, - stagehand, - useTextExtract, -}) => { - await stagehand.page.goto( - "https://browserbase.github.io/stagehand-eval-sites/sites/aigrant/", - ); - const companyList = await stagehand.page.extract({ - instruction: - "Extract all companies that received the AI grant and group them with their batch numbers as an array of objects. Each object should contain the company name and its corresponding batch number.", - schema: z.object({ - companies: z.array( - z.object({ - company: z.string(), - batch: z.string(), - }), - ), - }), - useTextExtract, - }); - - await stagehand.close(); - const companies = companyList.companies; - const expectedLength = 91; - - const expectedFirstItem = { - company: "Goodfire", - batch: "4", - }; - - const expectedLastItem = { - company: "Forefront", - batch: "1", - }; - - if (companies.length !== expectedLength) { - logger.error({ - message: "Incorrect number of companies extracted", - level: 0, - auxiliary: { - expected: { - value: expectedLength.toString(), - type: "integer", - }, - actual: { - value: companies.length.toString(), - type: "integer", - }, - }, - }); - return { - _success: false, - error: "Incorrect number of companies extracted", - logs: logger.getLogs(), - debugUrl, - sessionUrl, - }; - } - const firstItemMatches = - companies[0].company === expectedFirstItem.company && - companies[0].batch === expectedFirstItem.batch; - - if (!firstItemMatches) { - logger.error({ - message: "First company extracted does not match expected", - level: 0, - auxiliary: { - expected: { - value: JSON.stringify(expectedFirstItem), - type: "object", - }, - actual: { - value: JSON.stringify(companies[0]), - type: "object", - }, - }, - }); - return { - _success: false, - error: "First company extracted does not match expected", - logs: logger.getLogs(), - debugUrl, - sessionUrl, - }; - } - - const lastItemMatches = - companies[companies.length - 1].company === expectedLastItem.company && - companies[companies.length - 1].batch === expectedLastItem.batch; - - if (!lastItemMatches) { - logger.error({ - message: "Last company extracted does not match expected", - level: 0, - auxiliary: { - expected: { - value: JSON.stringify(expectedLastItem), - type: "object", - }, - actual: { - value: JSON.stringify(companies[companies.length - 1]), - type: "object", - }, - }, - }); - return { - _success: false, - error: "Last company extracted does not match expected", - logs: logger.getLogs(), - debugUrl, - sessionUrl, - }; - } - - return { - _success: true, - logs: logger.getLogs(), - debugUrl, - sessionUrl, - }; -}; diff --git a/evals/tasks/extract_aigrant_targeted.ts b/evals/tasks/extract_aigrant_targeted.ts deleted file mode 100644 index d72328505..000000000 --- a/evals/tasks/extract_aigrant_targeted.ts +++ /dev/null @@ -1,63 +0,0 @@ -import { z } from "zod"; -import { EvalFunction } from "@/types/evals"; - -export const extract_aigrant_targeted: EvalFunction = async ({ - logger, - useTextExtract, - debugUrl, - sessionUrl, - stagehand, -}) => { - await stagehand.page.goto( - "https://browserbase.github.io/stagehand-eval-sites/sites/aigrant/", - ); - const selector = "/html/body/div/ul[5]/li[28]"; - const company = await stagehand.page.extract({ - instruction: "Extract the company name.", - schema: z.object({ - company_name: z.string(), - }), - useTextExtract, - selector: selector, - }); - - await stagehand.close(); - const companyName = company.company_name; - - const expectedName = { - company_name: "Coframe", - }; - - const nameMatches = companyName == expectedName.company_name; - - if (!nameMatches) { - logger.error({ - message: "extracted company name does not match expected", - level: 0, - auxiliary: { - expected: { - value: expectedName.company_name, - type: "string", - }, - actual: { - value: companyName, - type: "string", - }, - }, - }); - return { - _success: false, - error: "Company name does not match expected", - logs: logger.getLogs(), - debugUrl, - sessionUrl, - }; - } - - return { - _success: true, - logs: logger.getLogs(), - debugUrl, - sessionUrl, - }; -}; diff --git a/evals/tasks/extract_aigrant_targeted_2.ts b/evals/tasks/extract_aigrant_targeted_2.ts deleted file mode 100644 index 772d2f4a4..000000000 --- a/evals/tasks/extract_aigrant_targeted_2.ts +++ /dev/null @@ -1,70 +0,0 @@ -import { z } from "zod"; -import { EvalFunction } from "@/types/evals"; - -export const extract_aigrant_targeted_2: EvalFunction = async ({ - logger, - useTextExtract, - debugUrl, - sessionUrl, - stagehand, -}) => { - await stagehand.page.goto( - "https://browserbase.github.io/stagehand-eval-sites/sites/aigrant/", - ); - const selector = "/html/body/div/ul[5]/li[28]"; - const company = await stagehand.page.extract({ - instruction: "Extract the name of the company that comes after 'Coframe'.", - schema: z.object({ - company_name: z.string(), - }), - useTextExtract, - selector: selector, - }); - - await stagehand.close(); - const companyName = company.company_name; - - // nameWeShouldNotGet matches the name of the company that comes after - // CoFrame on the website. Since we are using targeted_extract here, - // and passing in a selector that does NOT contain the nameWeShouldNotGet, - // the LLM should have no visibility into what comes after 'CoFrame' if - // targeted_extract is performing correctly - const nameWeShouldNotGet = { - company_name: "OpusClip", - }; - - const nameMatches = companyName == nameWeShouldNotGet.company_name; - - if (nameMatches) { - logger.error({ - message: - "extracted company name matches the company name that we SHOULD NOT get", - level: 0, - auxiliary: { - expected: { - value: nameWeShouldNotGet.company_name, - type: "string", - }, - actual: { - value: companyName, - type: "string", - }, - }, - }); - return { - _success: false, - error: - "extracted company name matches the company name that we SHOULD NOT get", - logs: logger.getLogs(), - debugUrl, - sessionUrl, - }; - } - - return { - _success: true, - logs: logger.getLogs(), - debugUrl, - sessionUrl, - }; -}; diff --git a/evals/tasks/extract_apartments.ts b/evals/tasks/extract_apartments.ts deleted file mode 100644 index 96cfd06b7..000000000 --- a/evals/tasks/extract_apartments.ts +++ /dev/null @@ -1,62 +0,0 @@ -import { z } from "zod"; -import { EvalFunction } from "../../types/evals"; - -export const extract_apartments: EvalFunction = async ({ - logger, - useTextExtract, - debugUrl, - sessionUrl, - stagehand, -}) => { - await stagehand.page.goto( - "https://www.apartments.com/san-francisco-ca/2-bedrooms/", - ); - const apartment_listings = await stagehand.page.extract({ - instruction: - "Extract all the apartment listings with their prices and their addresses.", - schema: z.object({ - listings: z.array( - z.object({ - price: z.string().describe("The price of the listing"), - trails: z.string().describe("The address of the listing"), - }), - ), - }), - useTextExtract, - }); - - await stagehand.close(); - const listings = apartment_listings.listings; - const expectedLength = 40; - - if (listings.length < expectedLength) { - logger.error({ - message: "Incorrect number of listings extracted", - level: 0, - auxiliary: { - expected: { - value: expectedLength.toString(), - type: "integer", - }, - actual: { - value: listings.length.toString(), - type: "integer", - }, - }, - }); - return { - _success: false, - error: "Incorrect number of listings extracted", - logs: logger.getLogs(), - debugUrl, - sessionUrl, - }; - } - - return { - _success: true, - logs: logger.getLogs(), - debugUrl, - sessionUrl, - }; -}; diff --git a/evals/tasks/extract_area_codes.ts b/evals/tasks/extract_area_codes.ts deleted file mode 100644 index 07f99ec32..000000000 --- a/evals/tasks/extract_area_codes.ts +++ /dev/null @@ -1,153 +0,0 @@ -import { EvalFunction } from "@/types/evals"; -import { z } from "zod"; - -export const extract_area_codes: EvalFunction = async ({ - logger, - useTextExtract, - debugUrl, - sessionUrl, - stagehand, -}) => { - await stagehand.page.goto( - "https://browserbase.github.io/stagehand-eval-sites/sites/ncc-area-codes/", - { waitUntil: "domcontentloaded" }, - ); - - const result = await stagehand.page.extract({ - instruction: - "Extract ALL the Primary Center names and their corresponding Area Code, and the name of their corresponding Zone.", - schema: z.object({ - primary_center_list: z.array( - z.object({ - zone_name: z - .string() - .describe( - "The name of the Zone that the Primary Center is in. For example, 'North Central Zone'.", - ), - primary_center_name: z - .string() - .describe( - "The name of the Primary Center. I.e., this is the name of the city or town.", - ), - area_code: z - .string() - .describe( - "The area code for the Primary Center. This will either be 2 or 3 digits.", - ), - }), - ), - }), - useTextExtract, - }); - - await stagehand.close(); - - const primaryCenterList = result.primary_center_list; - const expectedLength = 56; - - const expectedFirstItem = { - zone_name: "Lagos Zone", - primary_center_name: "Lagos", - area_code: "01", - }; - - const expectedLastItem = { - zone_name: "South-East", - primary_center_name: "Yenagoa", - area_code: "089", - }; - - if (primaryCenterList.length !== expectedLength) { - logger.error({ - message: "Incorrect number of primary centers extracted", - level: 0, - auxiliary: { - expected: { - value: expectedLength.toString(), - type: "integer", - }, - actual: { - value: primaryCenterList.length.toString(), - type: "integer", - }, - }, - }); - return { - _success: false, - error: "Incorrect number of primary centers extracted", - logs: logger.getLogs(), - debugUrl, - sessionUrl, - }; - } - const firstItemMatches = - primaryCenterList[0].zone_name === expectedFirstItem.zone_name && - primaryCenterList[0].primary_center_name === - expectedFirstItem.primary_center_name && - primaryCenterList[0].area_code === expectedFirstItem.area_code; - - if (!firstItemMatches) { - logger.error({ - message: "First primary center extracted does not match expected", - level: 0, - auxiliary: { - expected: { - value: JSON.stringify(expectedFirstItem), - type: "object", - }, - actual: { - value: JSON.stringify(primaryCenterList[0]), - type: "object", - }, - }, - }); - return { - _success: false, - error: "First primary center extracted does not match expected", - logs: logger.getLogs(), - debugUrl, - sessionUrl, - }; - } - - const lastItemMatches = - primaryCenterList[primaryCenterList.length - 1].zone_name === - expectedLastItem.zone_name && - primaryCenterList[primaryCenterList.length - 1].primary_center_name === - expectedLastItem.primary_center_name && - primaryCenterList[primaryCenterList.length - 1].area_code === - expectedLastItem.area_code; - - if (!lastItemMatches) { - logger.error({ - message: "Last primary center extracted does not match expected", - level: 0, - auxiliary: { - expected: { - value: JSON.stringify(expectedLastItem), - type: "object", - }, - actual: { - value: JSON.stringify( - primaryCenterList[primaryCenterList.length - 1], - ), - type: "object", - }, - }, - }); - return { - _success: false, - error: "Last primary center extracted does not match expected", - logs: logger.getLogs(), - debugUrl, - sessionUrl, - }; - } - - return { - _success: true, - logs: logger.getLogs(), - debugUrl, - sessionUrl, - }; -}; diff --git a/evals/tasks/extract_baptist_health.ts b/evals/tasks/extract_baptist_health.ts deleted file mode 100644 index 1bb90d83e..000000000 --- a/evals/tasks/extract_baptist_health.ts +++ /dev/null @@ -1,98 +0,0 @@ -import { EvalFunction } from "@/types/evals"; -import { compareStrings } from "@/evals/utils"; -import { z } from "zod"; - -export const extract_baptist_health: EvalFunction = async ({ - logger, - useTextExtract, - debugUrl, - sessionUrl, - stagehand, -}) => { - await stagehand.page.goto( - "https://browserbase.github.io/stagehand-eval-sites/sites/baptist-health/", - ); - - const result = await stagehand.page.extract({ - instruction: - "Extract the address, phone number, and fax number of the healthcare location.", - schema: z.object({ - address: z.string(), - phone: z.string(), - fax: z.string(), - }), - useTextExtract, - }); - - await stagehand.close(); - - const { address, phone, fax } = result; - const expected = { - address: "2055 East South Blvd; Suite 908 Montgomery, AL 36116", - phone: "334-747-2273", - fax: "334-747-7501", - }; - - const similarityThreshold = 0.85; - const failedFields: Array<{ - field: string; - similarity: number; - expected: string; - actual: string; - }> = []; - - const compareField = ( - actualVal: string, - expectedVal: string, - fieldName: string, - ) => { - const { similarity, meetsThreshold } = compareStrings( - actualVal, - expectedVal, - similarityThreshold, - ); - - if (!meetsThreshold) { - failedFields.push({ - field: fieldName, - similarity, - expected: expectedVal, - actual: actualVal, - }); - logger.error({ - message: `${fieldName} extracted does not meet similarity threshold`, - level: 0, - auxiliary: { - field: { value: fieldName, type: "string" }, - similarity: { value: similarity.toFixed(2), type: "string" }, - expected: { value: expectedVal, type: "string" }, - actual: { value: actualVal, type: "string" }, - }, - }); - } - - return meetsThreshold; - }; - - const addressOk = compareField(address, expected.address, "Address"); - const phoneOk = compareField(phone, expected.phone, "Phone number"); - const faxOk = compareField(fax, expected.fax, "Fax number"); - - if (!addressOk || !phoneOk || !faxOk) { - return { - _success: false, - error: "Some fields did not meet similarity threshold", - logs: logger.getLogs(), - debugUrl, - sessionUrl, - failedFields, - }; - } - - return { - _success: true, - logs: logger.getLogs(), - debugUrl, - sessionUrl, - }; -}; diff --git a/evals/tasks/extract_capacitor_info.ts b/evals/tasks/extract_capacitor_info.ts deleted file mode 100644 index f416b6a75..000000000 --- a/evals/tasks/extract_capacitor_info.ts +++ /dev/null @@ -1,112 +0,0 @@ -import { EvalFunction } from "@/types/evals"; -import { normalizeString } from "@/evals/utils"; -import { z } from "zod"; - -export const extract_capacitor_info: EvalFunction = async ({ - logger, - useTextExtract, - debugUrl, - sessionUrl, - stagehand, -}) => { - await stagehand.page.goto( - "https://browserbase.github.io/stagehand-eval-sites/sites/capacitor/", - ); - - const result = await stagehand.page.extract({ - instruction: "Extract the ECCN Code, RoHS Status, and Impedance.", - schema: z.object({ - ECCN_code: z.string(), - RoHS_Status: z.string(), - Impedance: z.string(), - }), - useTextExtract, - }); - - const { ECCN_code, RoHS_Status, Impedance } = result; - - const expected = { - ECCN_code: "EAR99", - RoHS_Status: "RoHS Compliant", - Impedance: "12mOhm", - }; - - if (normalizeString(ECCN_code) !== normalizeString(expected.ECCN_code)) { - logger.error({ - message: "ECCN code extracted does not match expected", - level: 0, - auxiliary: { - expected: { - value: normalizeString(expected.ECCN_code), - type: "string", - }, - actual: { - value: normalizeString(ECCN_code), - type: "string", - }, - }, - }); - return { - _success: false, - error: "ECCN code extracted does not match expected", - logs: logger.getLogs(), - debugUrl, - sessionUrl, - }; - } - - if (normalizeString(RoHS_Status) !== normalizeString(expected.RoHS_Status)) { - logger.error({ - message: "RoHS Status extracted does not match expected", - level: 0, - auxiliary: { - expected: { - value: normalizeString(expected.RoHS_Status), - type: "string", - }, - actual: { - value: normalizeString(RoHS_Status), - type: "string", - }, - }, - }); - return { - _success: false, - error: "RoHS Status extracted does not match expected", - logs: logger.getLogs(), - debugUrl, - sessionUrl, - }; - } - - if (normalizeString(Impedance) !== normalizeString(expected.Impedance)) { - logger.error({ - message: "Impedance extracted does not match expected", - level: 0, - auxiliary: { - expected: { - value: normalizeString(expected.Impedance), - type: "string", - }, - actual: { - value: normalizeString(Impedance), - type: "string", - }, - }, - }); - return { - _success: false, - error: "Impedance extracted does not match expected", - logs: logger.getLogs(), - debugUrl, - sessionUrl, - }; - } - - return { - _success: true, - logs: logger.getLogs(), - debugUrl, - sessionUrl, - }; -}; diff --git a/evals/tasks/extract_collaborators.ts b/evals/tasks/extract_collaborators.ts deleted file mode 100644 index 8e10ee020..000000000 --- a/evals/tasks/extract_collaborators.ts +++ /dev/null @@ -1,70 +0,0 @@ -import { EvalFunction } from "@/types/evals"; -import { z } from "zod"; - -export const extract_collaborators: EvalFunction = async ({ - logger, - useTextExtract, - debugUrl, - sessionUrl, - stagehand, -}) => { - try { - await stagehand.page.goto("https://github.com/facebook/react"); - await stagehand.page.act({ - action: "find and click the contributors section", - }); - - await stagehand.page.waitForLoadState("domcontentloaded"); - await stagehand.page.waitForLoadState("networkidle"); - await stagehand.page.waitForTimeout(5000); - - const { contributors } = await stagehand.page.extract({ - instruction: "Extract top 5 contributors of this repository", - schema: z.object({ - contributors: z.array( - z.object({ - github_username: z - .string() - .describe("the github username of the contributor"), - commits: z.number().describe("number of commits contributed"), - }), - ), - }), - useTextExtract, - }); - - await stagehand.close(); - - const EXPECTED_CONTRIBUTORS = [ - "zpao", - "gaearon", - "sebmarkbage", - "acdlite", - "sophiebits", - ]; - return { - _success: - contributors.length === EXPECTED_CONTRIBUTORS.length && - contributors.every( - (c, i) => - EXPECTED_CONTRIBUTORS[i] === c.github_username && c.commits >= 1000, - ), - contributors, - debugUrl, - sessionUrl, - logs: logger.getLogs(), - }; - } catch (error) { - console.error("Error or timeout occurred:", error); - - await stagehand.close(); - - return { - _success: false, - error: JSON.parse(JSON.stringify(error, null, 2)), - debugUrl, - sessionUrl, - logs: logger.getLogs(), - }; - } -}; diff --git a/evals/tasks/extract_csa.ts b/evals/tasks/extract_csa.ts deleted file mode 100644 index 9609d1d5d..000000000 --- a/evals/tasks/extract_csa.ts +++ /dev/null @@ -1,148 +0,0 @@ -import { EvalFunction } from "@/types/evals"; -import { z } from "zod"; - -export const extract_csa: EvalFunction = async ({ - useTextExtract, - debugUrl, - sessionUrl, - stagehand, - logger, -}) => { - const page = stagehand.page; - await page.goto( - "https://browserbase.github.io/stagehand-eval-sites/sites/csa/", - ); - - const result = await page.extract({ - instruction: - "Extract all the publications on the page including the publication date, session type, publication type, and annotation", - schema: z.object({ - publications: z.array( - z.object({ - publication_date: z.string(), - session_type: z.string(), - publication_type: z.string(), - annotation: z.string(), - }), - ), - }), - useTextExtract, - }); - - await stagehand.close(); - - const publications = result.publications; - const expectedLength = 14; - - const expectedFirstItem = { - publication_date: "11-30-2024", - session_type: "Regular Session", - publication_type: "Assembly Weekly History", - annotation: - "2024 -- This publication includes the complete histories of second-year bills. The complete electronic history of all bills is always available at leginfo.legislature.ca.gov", - }; - - const expectedLastItem = { - publication_date: "11-30-2016", - session_type: "1st Extraordinary Session", - publication_type: "Assembly Weekly History", - annotation: "", - }; - - if (publications.length < expectedLength) { - logger.error({ - message: "Incorrect number of publications extracted", - level: 0, - auxiliary: { - expected: { - value: `>= ${expectedLength}`, - type: "integer", - }, - actual: { - value: publications.length.toString(), - type: "integer", - }, - }, - }); - return { - _success: false, - error: "Incorrect number of publications extracted", - logs: logger.getLogs(), - debugUrl, - sessionUrl, - }; - } - - const hasExpectedFirstItem = publications.some((publication) => { - return ( - publication.publication_date === expectedFirstItem.publication_date && - publication.session_type === expectedFirstItem.session_type && - publication.publication_type === expectedFirstItem.publication_type && - publication.annotation === expectedFirstItem.annotation - ); - }); - - if (!hasExpectedFirstItem) { - logger.error({ - message: "Expected 'first' item not found in publications", - level: 0, - auxiliary: { - expected: { - value: JSON.stringify(expectedFirstItem), - type: "object", - }, - actual: { - value: JSON.stringify(publications), - type: "object", - }, - }, - }); - return { - _success: false, - error: "Expected 'first' item not found in publications", - logs: logger.getLogs(), - debugUrl, - sessionUrl, - }; - } - - const hasExpectedLastItem = publications.some((publication) => { - return ( - publication.publication_date === expectedLastItem.publication_date && - publication.session_type === expectedLastItem.session_type && - publication.publication_type === expectedLastItem.publication_type && - publication.annotation === expectedLastItem.annotation - ); - }); - - if (!hasExpectedLastItem) { - logger.error({ - message: "Expected 'last' item not found in publications", - level: 0, - auxiliary: { - expected: { - value: JSON.stringify(expectedLastItem), - type: "object", - }, - actual: { - value: JSON.stringify(publications), - type: "object", - }, - }, - }); - return { - _success: false, - error: "Expected 'last' item not found in publications", - logs: logger.getLogs(), - debugUrl, - sessionUrl, - }; - } - - return { - _success: true, - logs: logger.getLogs(), - debugUrl, - sessionUrl, - }; -}; diff --git a/evals/tasks/extract_geniusee.ts b/evals/tasks/extract_geniusee.ts deleted file mode 100644 index 18664ad6d..000000000 --- a/evals/tasks/extract_geniusee.ts +++ /dev/null @@ -1,65 +0,0 @@ -import { z } from "zod"; -import { EvalFunction } from "@/types/evals"; - -export const extract_geniusee: EvalFunction = async ({ - logger, - useTextExtract, - debugUrl, - sessionUrl, - stagehand, -}) => { - await stagehand.page.goto( - "https://browserbase.github.io/stagehand-eval-sites/sites/geniusee/", - ); - const selector = "/html/body/main/div[2]/div[2]/div[2]/table"; - const scalability = await stagehand.page.extract({ - instruction: - "Extract the scalability comment in the table for Gemini (Google)", - schema: z.object({ - scalability: z.string(), - }), - useTextExtract, - selector: selector, - }); - - await stagehand.close(); - const scalabilityComment = scalability.scalability; - - const expectedScalabilityComment = { - scalability: "Scalable architecture with API access", - }; - - const commentMatches = - scalabilityComment == expectedScalabilityComment.scalability; - - if (!commentMatches) { - logger.error({ - message: "extracted scalability comment does not match expected", - level: 0, - auxiliary: { - expected: { - value: expectedScalabilityComment.scalability, - type: "string", - }, - actual: { - value: scalabilityComment, - type: "string", - }, - }, - }); - return { - _success: false, - error: "extracted scalability comment does not match expected", - logs: logger.getLogs(), - debugUrl, - sessionUrl, - }; - } - - return { - _success: true, - logs: logger.getLogs(), - debugUrl, - sessionUrl, - }; -}; diff --git a/evals/tasks/extract_geniusee_2.ts b/evals/tasks/extract_geniusee_2.ts deleted file mode 100644 index 4a3f8006b..000000000 --- a/evals/tasks/extract_geniusee_2.ts +++ /dev/null @@ -1,72 +0,0 @@ -import { z } from "zod"; -import { EvalFunction } from "@/types/evals"; - -export const extract_geniusee_2: EvalFunction = async ({ - logger, - useTextExtract, - debugUrl, - sessionUrl, - stagehand, -}) => { - await stagehand.page.goto( - "https://browserbase.github.io/stagehand-eval-sites/sites/geniusee/", - ); - const selector = "/html/body/main/div[2]/div[2]/div[2]/table/tbody/tr[9]"; - const scalability = await stagehand.page.extract({ - instruction: - "Extract the scalability comment in the table for Gemini (Google)", - schema: z.object({ - scalability: z.string(), - }), - useTextExtract, - selector: selector, - }); - - await stagehand.close(); - const scalabilityComment = scalability.scalability; - - // scalabilityCommentWeShouldNotGet matches a scalability comment in the table, - // but since we are using targeted_extract here, - // and passing in a selector that does NOT contain the scalabilityCommentWeShouldNotGet, - // the LLM should have no visibility into scalabilityCommentWeShouldNotGet if - // targeted_extract is performing correctly - const scalabilityCommentWeShouldNotGet = { - scalability: "Scalable architecture with API access", - }; - - const commentMatches = - scalabilityComment == scalabilityCommentWeShouldNotGet.scalability; - - if (commentMatches) { - logger.error({ - message: - "extracted scalability comment matches the scalability comment that we SHOULD NOT get", - level: 0, - auxiliary: { - expected: { - value: scalabilityCommentWeShouldNotGet.scalability, - type: "string", - }, - actual: { - value: scalabilityComment, - type: "string", - }, - }, - }); - return { - _success: false, - error: - "scalability comment matches the scalability comment that we SHOULD NOT get", - logs: logger.getLogs(), - debugUrl, - sessionUrl, - }; - } - - return { - _success: true, - logs: logger.getLogs(), - debugUrl, - sessionUrl, - }; -}; diff --git a/evals/tasks/extract_github_commits.ts b/evals/tasks/extract_github_commits.ts deleted file mode 100644 index 4d8256345..000000000 --- a/evals/tasks/extract_github_commits.ts +++ /dev/null @@ -1,65 +0,0 @@ -import { EvalFunction } from "@/types/evals"; -import { z } from "zod"; - -export const extract_github_commits: EvalFunction = async ({ - logger, - useTextExtract, - debugUrl, - sessionUrl, - stagehand, -}) => { - try { - await stagehand.page.goto("https://github.com/facebook/react"); - - await stagehand.page.act({ - action: - "find commit history, generally described by the number of commits", - }); - const { commits } = await stagehand.page.extract({ - instruction: "Extract last 20 commits", - schema: z.object({ - commits: z.array( - z.object({ - commit_message: z.string(), - commit_url: z.string(), - commit_hash: z.string(), - }), - ), - }), - useTextExtract, - }); - - logger.log({ - message: "Extracted commits", - level: 1, - auxiliary: { - commits: { - value: JSON.stringify(commits), - type: "object", - }, - }, - }); - - await stagehand.close(); - - return { - _success: commits.length === 20, - commits, - debugUrl, - sessionUrl, - logs: logger.getLogs(), - }; - } catch (error) { - console.error("Error or timeout occurred:", error); - - await stagehand.close(); - - return { - _success: false, - error: JSON.parse(JSON.stringify(error, null, 2)), - debugUrl, - sessionUrl, - logs: logger.getLogs(), - }; - } -}; diff --git a/evals/tasks/extract_github_stars.ts b/evals/tasks/extract_github_stars.ts deleted file mode 100644 index 7ab994076..000000000 --- a/evals/tasks/extract_github_stars.ts +++ /dev/null @@ -1,56 +0,0 @@ -import { EvalFunction } from "@/types/evals"; -import { z } from "zod"; - -export const extract_github_stars: EvalFunction = async ({ - logger, - useTextExtract, - debugUrl, - sessionUrl, - stagehand, -}) => { - try { - await stagehand.page.goto("https://github.com/facebook/react"); - - const { stars } = await stagehand.page.extract({ - instruction: "Extract the number of stars for the project", - schema: z.object({ - stars: z.number().describe("the number of stars for the project"), - }), - useTextExtract, - }); - - const expectedStarsString = await stagehand.page - .locator("#repo-stars-counter-star") - .first() - .innerHTML(); - - const expectedStars = expectedStarsString.toLowerCase().endsWith("k") - ? parseFloat(expectedStarsString.slice(0, -1)) * 1000 - : parseFloat(expectedStarsString); - - const tolerance = 1000; - const isWithinTolerance = Math.abs(stars - expectedStars) <= tolerance; - - await stagehand.close(); - - return { - _success: isWithinTolerance, - stars, - debugUrl, - sessionUrl, - logs: logger.getLogs(), - }; - } catch (error) { - console.error("Error or timeout occurred:", error); - - await stagehand.close(); - - return { - _success: false, - error: JSON.parse(JSON.stringify(error, null, 2)), - debugUrl, - sessionUrl, - logs: logger.getLogs(), - }; - } -}; diff --git a/evals/tasks/extract_hamilton_weather.ts b/evals/tasks/extract_hamilton_weather.ts deleted file mode 100644 index 683c14c65..000000000 --- a/evals/tasks/extract_hamilton_weather.ts +++ /dev/null @@ -1,74 +0,0 @@ -import { EvalFunction } from "@/types/evals"; -import { z } from "zod"; - -export const extract_hamilton_weather: EvalFunction = async ({ - logger, - useTextExtract, - debugUrl, - sessionUrl, - stagehand, -}) => { - try { - await stagehand.page.goto( - "https://browserbase.github.io/stagehand-eval-sites/sites/hamilton-weather/", - ); - const xpath = - "/html/body[1]/div[5]/main[1]/article[1]/div[6]/div[2]/div[1]/table[1]"; - - const weatherData = await stagehand.page.extract({ - instruction: "extract the weather data for Sun, Feb 23 at 11PM", - schema: z.object({ - temperature: z.string(), - weather_description: z.string(), - wind: z.string(), - humidity: z.string(), - barometer: z.string(), - visibility: z.string(), - }), - useTextExtract, - selector: xpath, - }); - - // Define the expected weather data - const expectedWeatherData = { - temperature: "27 °F", - weather_description: "Light snow. Overcast.", - wind: "6 mph", - humidity: "93%", - barometer: '30.07 "Hg', - visibility: "10 mi", - }; - - // Check that every field matches the expected value - const isWeatherCorrect = - weatherData.temperature === expectedWeatherData.temperature && - weatherData.weather_description === - expectedWeatherData.weather_description && - weatherData.wind === expectedWeatherData.wind && - weatherData.humidity === expectedWeatherData.humidity && - weatherData.barometer === expectedWeatherData.barometer && - weatherData.visibility === expectedWeatherData.visibility; - - await stagehand.close(); - - return { - _success: isWeatherCorrect, - weatherData, - debugUrl, - sessionUrl, - logs: logger.getLogs(), - }; - } catch (error) { - console.error("Error or timeout occurred:", error); - - await stagehand.close(); - - return { - _success: false, - error: JSON.parse(JSON.stringify(error, null, 2)), - debugUrl, - sessionUrl, - logs: logger.getLogs(), - }; - } -}; diff --git a/evals/tasks/extract_jfk_links.ts b/evals/tasks/extract_jfk_links.ts deleted file mode 100644 index dc735c49b..000000000 --- a/evals/tasks/extract_jfk_links.ts +++ /dev/null @@ -1,125 +0,0 @@ -import { EvalFunction } from "@/types/evals"; -import { z } from "zod"; - -export const extract_jfk_links: EvalFunction = async ({ - logger, - debugUrl, - sessionUrl, - stagehand, -}) => { - try { - await stagehand.page.goto( - "https://browserbase.github.io/stagehand-eval-sites/sites/jfk/", - ); - - const extraction = await stagehand.page.extract({ - instruction: - "extract all the record file name and their corresponding links", - schema: z.object({ - records: z.array( - z.object({ - file_name: z.string().describe("the file name of the record"), - link: z.string().url(), - }), - ), - }), - }); - - // The list of records we expect to see - const expectedRecords = [ - { - file_name: "104-10003-10041.pdf", - link: "https://www.archives.gov/files/research/jfk/releases/2025/0318/104-10003-10041.pdf", - }, - { - file_name: "104-10004-10143 (C06932208).pdf", - link: "https://www.archives.gov/files/research/jfk/releases/2025/0318/104-10004-10143%20(C06932208).pdf", - }, - { - file_name: "104-10004-10143.pdf", - link: "https://www.archives.gov/files/research/jfk/releases/2025/0318/104-10004-10143.pdf", - }, - { - file_name: "104-10004-10156.pdf", - link: "https://www.archives.gov/files/research/jfk/releases/2025/0318/104-10004-10156.pdf", - }, - { - file_name: "104-10004-10213.pdf", - link: "https://www.archives.gov/files/research/jfk/releases/2025/0318/104-10004-10213.pdf", - }, - { - file_name: "104-10005-10321.pdf", - link: "https://www.archives.gov/files/research/jfk/releases/2025/0318/104-10005-10321.pdf", - }, - { - file_name: "104-10006-10247.pdf", - link: "https://www.archives.gov/files/research/jfk/releases/2025/0318/104-10006-10247.pdf", - }, - { - file_name: "104-10007-10345.pdf", - link: "https://www.archives.gov/files/research/jfk/releases/2025/0318/104-10007-10345.pdf", - }, - { - file_name: "104-10009-10021.pdf", - link: "https://www.archives.gov/files/research/jfk/releases/2025/0318/104-10009-10021.pdf", - }, - { - file_name: "104-10009-10222.pdf", - link: "https://www.archives.gov/files/research/jfk/releases/2025/0318/104-10009-10222.pdf", - }, - ]; - - const extractedRecords = extraction.records; - - // Check that all expected records exist in the extraction - const missingRecords = expectedRecords.filter((expected) => { - return !extractedRecords.some( - (r) => r.file_name === expected.file_name && r.link === expected.link, - ); - }); - - // Check that the extraction array is exactly length 10 - if (extractedRecords.length !== 10) { - await stagehand.close(); - return { - _success: false, - reason: `Extraction has ${extractedRecords.length} records (expected 10).`, - debugUrl, - sessionUrl, - logs: logger.getLogs(), - }; - } - - if (missingRecords.length > 0) { - await stagehand.close(); - return { - _success: false, - reason: "Missing one or more expected records.", - missingRecords, - extractedRecords, - debugUrl, - sessionUrl, - logs: logger.getLogs(), - }; - } - - // If we reach here, the number of records is correct, and all are present - await stagehand.close(); - return { - _success: true, - debugUrl, - sessionUrl, - logs: logger.getLogs(), - }; - } catch (error) { - await stagehand.close(); - - return { - _success: false, - error: JSON.parse(JSON.stringify(error, null, 2)), - debugUrl, - sessionUrl, - logs: logger.getLogs(), - }; - } -}; diff --git a/evals/tasks/extract_jstor_news.ts b/evals/tasks/extract_jstor_news.ts deleted file mode 100644 index 4c5c690c9..000000000 --- a/evals/tasks/extract_jstor_news.ts +++ /dev/null @@ -1,136 +0,0 @@ -import { EvalFunction } from "@/types/evals"; -import { z } from "zod"; - -export const extract_jstor_news: EvalFunction = async ({ - logger, - useTextExtract, - debugUrl, - sessionUrl, - stagehand, -}) => { - await stagehand.page.goto( - "https://browserbase.github.io/stagehand-eval-sites/sites/jstor/", - { - waitUntil: "load", - }, - ); - await stagehand.page.act({ action: "close the cookie" }); - - const result = await stagehand.page.extract({ - instruction: "Extract ALL the news report titles and their dates.", - schema: z.object({ - reports: z.array( - z.object({ - report_name: z - .string() - .describe("The name or title of the news report."), - publish_date: z - .string() - .describe("The date the news report was published."), - }), - ), - }), - useTextExtract, - }); - - await stagehand.close(); - - const reports = result.reports; - const expectedLength = 10; - - const expectedFirstItem = { - report_name: "JSTOR retires Publisher Sales Service", - publish_date: "December 9, 2024", - }; - - const expectedLastItem = { - report_name: "Path to Open announces 2024 titles", - publish_date: "May 10, 2024", - }; - - if (reports.length !== expectedLength) { - logger.error({ - message: "Incorrect number of reports extracted", - level: 0, - auxiliary: { - expected: { - value: expectedLength.toString(), - type: "integer", - }, - actual: { - value: reports.length.toString(), - type: "integer", - }, - }, - }); - return { - _success: false, - error: "Incorrect number of reports extracted", - logs: logger.getLogs(), - debugUrl, - sessionUrl, - }; - } - const firstItemMatches = - reports[0].report_name === expectedFirstItem.report_name && - reports[0].publish_date === expectedFirstItem.publish_date; - - if (!firstItemMatches) { - logger.error({ - message: "First report extracted does not match expected", - level: 0, - auxiliary: { - expected: { - value: JSON.stringify(expectedFirstItem), - type: "object", - }, - actual: { - value: JSON.stringify(reports[0]), - type: "object", - }, - }, - }); - return { - _success: false, - error: "First report extracted does not match expected", - logs: logger.getLogs(), - debugUrl, - sessionUrl, - }; - } - - const lastItemMatches = - reports[reports.length - 1].report_name === expectedLastItem.report_name && - reports[reports.length - 1].publish_date === expectedLastItem.publish_date; - - if (!lastItemMatches) { - logger.error({ - message: "Last report extracted does not match expected", - level: 0, - auxiliary: { - expected: { - value: JSON.stringify(expectedLastItem), - type: "object", - }, - actual: { - value: JSON.stringify(reports[reports.length - 1]), - type: "object", - }, - }, - }); - return { - _success: false, - error: "Last report extracted does not match expected", - logs: logger.getLogs(), - debugUrl, - sessionUrl, - }; - } - - return { - _success: true, - logs: logger.getLogs(), - debugUrl, - sessionUrl, - }; -}; diff --git a/evals/tasks/extract_memorial_healthcare.ts b/evals/tasks/extract_memorial_healthcare.ts deleted file mode 100644 index c01dd5480..000000000 --- a/evals/tasks/extract_memorial_healthcare.ts +++ /dev/null @@ -1,185 +0,0 @@ -import { EvalFunction } from "@/types/evals"; -import { z } from "zod"; -import { compareStrings } from "@/evals/utils"; - -export const extract_memorial_healthcare: EvalFunction = async ({ - logger, - useTextExtract, - debugUrl, - sessionUrl, - stagehand, -}) => { - await stagehand.page.goto( - "https://browserbase.github.io/stagehand-eval-sites/sites/mycmh/", - ); - - const result = await stagehand.page.extract({ - instruction: - "extract a list of the first three healthcare centers on this page, with their name, full address, and phone number", - schema: z.object({ - health_centers: z.array( - z.object({ - name: z.string(), - phone_number: z.string(), - address: z.string(), - }), - ), - }), - useTextExtract, - }); - - await stagehand.close(); - - const health_centers: Array< - Partial<{ name: string; phone_number: string; address: string }> - > = result.health_centers; - - const expectedLength = 3; - const similarityThreshold = 0.85; - - const expectedFirstItem = { - name: "Community Memorial Breast Center", - phone_number: "805-948-5093", - address: "168 North Brent Street, Suite 401, Ventura, CA 93003", - }; - - const expectedLastItem = { - name: "Community Memorial Dermatology and Mohs Surgery", - phone_number: "805-948-6920", - address: "168 North Brent Street, Suite 403, Ventura, CA 93003", - }; - - if (health_centers.length !== expectedLength) { - logger.error({ - message: "Incorrect number of health centers extracted", - level: 0, - auxiliary: { - expected: { - value: expectedLength.toString(), - type: "integer", - }, - actual: { - value: health_centers.length.toString(), - type: "integer", - }, - }, - }); - - return { - _success: false, - error: "Incorrect number of health centers extracted", - logs: logger.getLogs(), - debugUrl, - sessionUrl, - }; - } - - const validateHealthCenter = ( - center: Partial<{ name: string; phone_number: string; address: string }>, - ): { name: string; phone_number: string; address: string } | null => { - if (center.name && center.phone_number && center.address) { - return center as { name: string; phone_number: string; address: string }; - } - logger.error({ - message: "Invalid health center data", - level: 0, - auxiliary: { - center: { value: JSON.stringify(center), type: "object" }, - }, - }); - return null; - }; - - const validHealthCenters = health_centers - .map(validateHealthCenter) - .filter(Boolean) as Array<{ - name: string; - phone_number: string; - address: string; - }>; - - if (validHealthCenters.length < expectedLength) { - return { - _success: false, - error: "One or more health centers have missing fields", - logs: logger.getLogs(), - debugUrl, - sessionUrl, - }; - } - - const compareField = ( - actual: string, - expected: string, - fieldName: string, - ): boolean => { - const { similarity, meetsThreshold } = compareStrings( - actual, - expected, - similarityThreshold, - ); - - if (!meetsThreshold) { - logger.error({ - message: `Field "${fieldName}" does not meet similarity threshold`, - level: 0, - auxiliary: { - field: { value: fieldName, type: "string" }, - similarity: { value: similarity.toFixed(2), type: "float" }, - expected: { value: expected, type: "string" }, - actual: { value: actual, type: "string" }, - }, - }); - } - - return meetsThreshold; - }; - - const compareItem = ( - actual: { name: string; phone_number: string; address: string }, - expected: { name: string; phone_number: string; address: string }, - position: string, - ): boolean => { - const fields = [ - { field: "name", actual: actual.name, expected: expected.name }, - { - field: "phone_number", - actual: actual.phone_number, - expected: expected.phone_number, - }, - { field: "address", actual: actual.address, expected: expected.address }, - ]; - - return fields.every(({ field, actual, expected }) => - compareField(actual, expected, `${position} ${field}`), - ); - }; - - const firstItemMatches = compareItem( - validHealthCenters[0], - expectedFirstItem, - "First", - ); - const lastItemMatches = compareItem( - validHealthCenters[validHealthCenters.length - 1], - expectedLastItem, - "Last", - ); - - if (!firstItemMatches || !lastItemMatches) { - return { - _success: false, - error: "One or more fields do not match expected values", - logs: logger.getLogs(), - debugUrl, - sessionUrl, - }; - } - - return { - _success: true, - logs: logger.getLogs(), - debugUrl, - sessionUrl, - }; -}; diff --git a/evals/tasks/extract_nhl_stats.ts b/evals/tasks/extract_nhl_stats.ts deleted file mode 100644 index fb9e982ac..000000000 --- a/evals/tasks/extract_nhl_stats.ts +++ /dev/null @@ -1,118 +0,0 @@ -import { EvalFunction } from "@/types/evals"; -import { normalizeString } from "@/evals/utils"; -import { z } from "zod"; - -export const extract_nhl_stats: EvalFunction = async ({ - debugUrl, - sessionUrl, - stagehand, - logger, - useTextExtract, -}) => { - await stagehand.page.goto( - "https://www.hockeydb.com/ihdb/stats/top_league.php?lid=nhl1927&sid=1990", - { - waitUntil: "domcontentloaded", - }, - ); - - const result = await stagehand.page.extract({ - instruction: - "Extract the name of the goal scoring leader, their number of goals they scored, and the team they played for.", - schema: z.object({ - name: z.string(), - num_goals: z.string(), - team: z.string(), - }), - useTextExtract, - }); - - await stagehand.close(); - - const { name, num_goals, team } = result; - - const expected = { - name: "Brett Hull", - num_goals: "72", - team: "St. Louis", - }; - - if (normalizeString(name) !== normalizeString(expected.name)) { - logger.error({ - message: "Player name extracted does not match expected", - level: 0, - auxiliary: { - expected: { - value: normalizeString(expected.name), - type: "string", - }, - actual: { - value: normalizeString(name), - type: "string", - }, - }, - }); - return { - _success: false, - error: "Player name extracted does not match expected", - logs: logger.getLogs(), - debugUrl, - sessionUrl, - }; - } - - if (normalizeString(num_goals) !== normalizeString(expected.num_goals)) { - logger.error({ - message: "Number of goals extracted does not match expected", - level: 0, - auxiliary: { - expected: { - value: normalizeString(expected.num_goals), - type: "string", - }, - actual: { - value: normalizeString(num_goals), - type: "string", - }, - }, - }); - return { - _success: false, - error: "Number of goals extracted does not match expected", - logs: logger.getLogs(), - debugUrl, - sessionUrl, - }; - } - - if (normalizeString(team) !== normalizeString(expected.team)) { - logger.error({ - message: "Player team extracted does not match expected", - level: 0, - auxiliary: { - expected: { - value: normalizeString(expected.team), - type: "string", - }, - actual: { - value: normalizeString(team), - type: "string", - }, - }, - }); - return { - _success: false, - error: "Player team extracted does not match expected", - logs: logger.getLogs(), - debugUrl, - sessionUrl, - }; - } - - return { - _success: true, - logs: logger.getLogs(), - debugUrl, - sessionUrl, - }; -}; diff --git a/evals/tasks/extract_partners.ts b/evals/tasks/extract_partners.ts deleted file mode 100644 index e1851802f..000000000 --- a/evals/tasks/extract_partners.ts +++ /dev/null @@ -1,94 +0,0 @@ -import { EvalFunction } from "@/types/evals"; -import { z } from "zod"; - -export const extract_partners: EvalFunction = async ({ - debugUrl, - sessionUrl, - stagehand, - logger, - useTextExtract, -}) => { - try { - await stagehand.page.goto("https://ramp.com"); - - await stagehand.page.act({ - action: "move down to the bottom of the page.", - }); - - await stagehand.page.act({ - action: "Close the popup.", - }); - - await stagehand.page.act({ - action: "Find and click on the link that leads to the partners page.", - }); - - const partners = await stagehand.page.extract({ - instruction: ` - Extract all of the partner categories on the page. - `, - schema: z.object({ - partners: z.array( - z.object({ - partner_category: z.string().describe("The partner category"), - }), - ), - explanation: z - .string() - .optional() - .describe("Any explanation about partner listing or absence thereof"), - }), - useTextExtract, - }); - - const expectedPartners = [ - "Accounting Partners", - "Private Equity & Venture Capital Partners", - "Services Partners", - "Affiliates", - ]; - - const foundPartners = partners.partners.map((partner) => - partner.partner_category.toLowerCase(), - ); - - const allExpectedPartnersFound = expectedPartners.every((partner) => - foundPartners.includes(partner.toLowerCase()), - ); - - await stagehand.close(); - - return { - _success: allExpectedPartnersFound, - partners, - debugUrl, - sessionUrl, - logs: logger.getLogs(), - }; - } catch (error) { - logger.error({ - message: "error in extractPartners function", - level: 0, - auxiliary: { - error: { - value: error.message, - type: "string", - }, - trace: { - value: error.stack, - type: "string", - }, - }, - }); - - await stagehand.close(); - - return { - _success: false, - debugUrl, - sessionUrl, - error: JSON.parse(JSON.stringify(error, null, 2)), - logs: logger.getLogs(), - }; - } -}; diff --git a/evals/tasks/extract_press_releases.ts b/evals/tasks/extract_press_releases.ts deleted file mode 100644 index b2fe815e3..000000000 --- a/evals/tasks/extract_press_releases.ts +++ /dev/null @@ -1,128 +0,0 @@ -import { EvalFunction } from "@/types/evals"; -import { z } from "zod"; -import { compareStrings } from "@/evals/utils"; - -export const extract_press_releases: EvalFunction = async ({ - debugUrl, - sessionUrl, - stagehand, - logger, - useTextExtract, -}) => { - const schema = z.object({ - items: z.array( - z.object({ - title: z.string().describe("The title of the press release"), - publish_date: z - .string() - .describe("The date the press release was published"), - }), - ), - }); - - type PressRelease = z.infer["items"][number]; - - try { - await stagehand.page.goto( - "https://browserbase.github.io/stagehand-eval-sites/sites/press-releases/", - { - waitUntil: "networkidle", - }, - ); - await new Promise((resolve) => setTimeout(resolve, 5000)); - - const rawResult = await stagehand.page.extract({ - instruction: - "extract the title and corresponding publish date of EACH AND EVERY press releases on this page. DO NOT MISS ANY PRESS RELEASES.", - schema, - useTextExtract, - }); - - const parsed = schema.parse(rawResult); - const { items } = parsed; - - await stagehand.close(); - - const expectedLength = 28; - const expectedFirstItem: PressRelease = { - title: "UAW Region 9A Endorses Brad Lander for Mayor", - publish_date: "Dec 4, 2024", - }; - const expectedLastItem: PressRelease = { - title: "Fox Sued by New York City Pension Funds Over Election Falsehoods", - publish_date: "Nov 12, 2023", - }; - - if (items.length <= expectedLength) { - logger.error({ - message: "Not enough items extracted", - level: 0, - auxiliary: { - expected: { - value: `> ${expectedLength}`, - type: "string", - }, - actual: { - value: items.length.toString(), - type: "integer", - }, - }, - }); - return { - _success: false, - error: "Not enough items extracted", - logs: logger.getLogs(), - debugUrl, - sessionUrl, - }; - } - - const isItemMatch = (item: PressRelease, expected: PressRelease) => { - const titleComparison = compareStrings(item.title, expected.title, 0.9); - const dateComparison = compareStrings( - item.publish_date, - expected.publish_date, - 0.9, - ); - return titleComparison.meetsThreshold && dateComparison.meetsThreshold; - }; - - const foundFirstItem = items.some((item) => - isItemMatch(item, expectedFirstItem), - ); - const foundLastItem = items.some((item) => - isItemMatch(item, expectedLastItem), - ); - - return { - _success: foundFirstItem && foundLastItem, - logs: logger.getLogs(), - debugUrl, - sessionUrl, - }; - } catch (error) { - logger.error({ - message: `Error in extract_press_releases function`, - level: 0, - auxiliary: { - error: { - value: (error as Error).message || JSON.stringify(error), - type: "string", - }, - trace: { - value: (error as Error).stack, - type: "string", - }, - }, - }); - return { - _success: false, - error: "An error occurred during extraction", - logs: logger.getLogs(), - debugUrl, - sessionUrl, - }; - } finally { - await stagehand.context.close(); - } -}; diff --git a/evals/tasks/extract_professional_info.ts b/evals/tasks/extract_professional_info.ts deleted file mode 100644 index c77708f5c..000000000 --- a/evals/tasks/extract_professional_info.ts +++ /dev/null @@ -1,123 +0,0 @@ -import { EvalFunction } from "@/types/evals"; -import { normalizeString } from "@/evals/utils"; -import { z } from "zod"; - -export const extract_professional_info: EvalFunction = async ({ - debugUrl, - sessionUrl, - stagehand, - logger, - useTextExtract, -}) => { - await stagehand.page.goto( - "https://browserbase.github.io/stagehand-eval-sites/sites/professional-info/", - ); - - const result = await stagehand.page.extract({ - instruction: - "Extract the list of Practices, phone number, and fax number of the professional.", - schema: z.object({ - practices: z.array(z.string()), - phone: z.string(), - fax: z.string(), - }), - useTextExtract, - }); - - await stagehand.close(); - - const { practices, phone, fax } = result; - - const expected = { - practices: [ - "Restructuring", - "Finance", - "Hybrid Capital & Special Situations", - "Private Credit", - ], - phone: "+1-212-373-3262", - fax: "+1-212-492-0262", - }; - - if ( - JSON.stringify(practices.map(normalizeString)) !== - JSON.stringify(expected.practices.map(normalizeString)) - ) { - logger.error({ - message: "Practices extracted do not match expected", - level: 0, - auxiliary: { - expected: { - value: JSON.stringify(expected.practices), - type: "object", - }, - actual: { - value: JSON.stringify(practices), - type: "object", - }, - }, - }); - return { - _success: false, - error: "Practices extracted do not match expected", - logs: logger.getLogs(), - debugUrl, - sessionUrl, - }; - } - - if (normalizeString(phone) !== normalizeString(expected.phone)) { - logger.error({ - message: "Phone number extracted does not match expected", - level: 0, - auxiliary: { - expected: { - value: normalizeString(expected.phone), - type: "string", - }, - actual: { - value: normalizeString(phone), - type: "string", - }, - }, - }); - return { - _success: false, - error: "Phone number extracted does not match expected", - logs: logger.getLogs(), - debugUrl, - sessionUrl, - }; - } - - if (normalizeString(fax) !== normalizeString(expected.fax)) { - logger.error({ - message: "Fax number extracted does not match expected", - level: 0, - auxiliary: { - expected: { - value: normalizeString(expected.fax), - type: "string", - }, - actual: { - value: normalizeString(fax), - type: "string", - }, - }, - }); - return { - _success: false, - error: "Fax number extracted does not match expected", - logs: logger.getLogs(), - debugUrl, - sessionUrl, - }; - } - - return { - _success: true, - logs: logger.getLogs(), - debugUrl, - sessionUrl, - }; -}; diff --git a/evals/tasks/extract_public_notices.ts b/evals/tasks/extract_public_notices.ts deleted file mode 100644 index 73a76864c..000000000 --- a/evals/tasks/extract_public_notices.ts +++ /dev/null @@ -1,172 +0,0 @@ -import { EvalFunction } from "@/types/evals"; -import { z } from "zod"; -import { compareStrings } from "@/evals/utils"; - -export const extract_public_notices: EvalFunction = async ({ - debugUrl, - sessionUrl, - stagehand, - logger, - useTextExtract, -}) => { - await stagehand.page.goto( - "https://browserbase.github.io/stagehand-eval-sites/sites/sars/", - { waitUntil: "load" }, - ); - - const result = await stagehand.page.extract({ - instruction: - "Extract ALL the public notice descriptions with their corresponding, GG number and publication date. Extract ALL notices from 2024 through 2020. Do not include the Notice number.", - schema: z.object({ - public_notices: z.array( - z.object({ - notice_description: z - .string() - .describe( - "the description of the notice. Do not include the Notice number", - ), - gg_number: z - .string() - .describe("the GG number of the notice. For example, GG 12345"), - publication_date: z - .string() - .describe( - "the publication date of the notice. For example, 8 December 2021", - ), - }), - ), - }), - useTextExtract, - }); - - await stagehand.close(); - - const publicNotices = result.public_notices; - const expectedLength = 24; - - const expectedFirstItem = { - notice_description: - "Additional considerations in terms of section 80(2) in respect of which an application for a binding private ruling or a binding class ruling may be rejected", - gg_number: "GG 51526", - publication_date: "8 November 2024", - }; - - const expectedLastItem = { - notice_description: - "Notice in terms of section 25, read with section 66(1) of the Income Tax Act, 1962, for submission of 2020 income tax returns", - gg_number: "GG 43495", - publication_date: "3 July 2020", - }; - - if (publicNotices.length !== expectedLength) { - logger.error({ - message: "Incorrect number of public notices extracted", - level: 0, - auxiliary: { - expected: { - value: expectedLength.toString(), - type: "integer", - }, - actual: { - value: publicNotices.length.toString(), - type: "integer", - }, - }, - }); - return { - _success: false, - error: "Incorrect number of public notices extracted", - logs: logger.getLogs(), - debugUrl, - sessionUrl, - }; - } - const firstItemMatches = - compareStrings( - publicNotices[0].notice_description, - expectedFirstItem.notice_description, - 0.9, - ) && - compareStrings( - publicNotices[0].gg_number, - expectedFirstItem.gg_number, - 0.9, - ) && - compareStrings( - publicNotices[0].publication_date, - expectedFirstItem.publication_date, - 0.9, - ); - - if (!firstItemMatches) { - logger.error({ - message: "First public notice extracted does not match expected", - level: 0, - auxiliary: { - expected: { - value: JSON.stringify(expectedFirstItem), - type: "object", - }, - actual: { - value: JSON.stringify(publicNotices[0]), - type: "object", - }, - }, - }); - return { - _success: false, - error: "First public notice extracted does not match expected", - logs: logger.getLogs(), - debugUrl, - sessionUrl, - }; - } - - const lastItemMatches = - compareStrings( - publicNotices[publicNotices.length - 1].notice_description, - expectedLastItem.notice_description, - 0.9, - ) && - compareStrings( - publicNotices[publicNotices.length - 1].gg_number, - expectedLastItem.gg_number, - 0.9, - ) && - compareStrings( - publicNotices[publicNotices.length - 1].publication_date, - expectedLastItem.publication_date, - 0.9, - ); - - if (!lastItemMatches) { - logger.error({ - message: "Last public notice extracted does not match expected", - level: 0, - auxiliary: { - expected: { - value: JSON.stringify(expectedLastItem), - type: "object", - }, - actual: { - value: JSON.stringify(publicNotices[publicNotices.length - 1]), - type: "object", - }, - }, - }); - return { - _success: false, - error: "Last public notice extracted does not match expected", - logs: logger.getLogs(), - debugUrl, - sessionUrl, - }; - } - - return { - _success: true, - logs: logger.getLogs(), - debugUrl, - sessionUrl, - }; -}; diff --git a/evals/tasks/extract_recipe.ts b/evals/tasks/extract_recipe.ts deleted file mode 100644 index 1e5040688..000000000 --- a/evals/tasks/extract_recipe.ts +++ /dev/null @@ -1,96 +0,0 @@ -import { EvalFunction } from "@/types/evals"; -import { z } from "zod"; - -export const extract_recipe: EvalFunction = async ({ - debugUrl, - sessionUrl, - stagehand, - logger, - useTextExtract, -}) => { - await stagehand.page.goto( - "https://browserbase.github.io/stagehand-eval-sites/sites/allrecipes-extract/", - { - waitUntil: "domcontentloaded", - }, - ); - - const selector = "/html/body/main/article/div[3]/div[3]/div[4]"; - const recipeDetails = await stagehand.page.extract({ - instruction: - "Extract the title of the number of tablespoons of olive oil needed for the steak, and the number of teaspoons of lemon juice needed for the mushroom pan sauce.", - schema: z.object({ - tablespoons_olive_oil: z - .number() - .describe( - "the number of tablespoons of olive oil needed for the steak", - ), - teaspoons_lemon_juice: z - .number() - .describe( - "the number of teaspoons of lemon juice needed for the mushroom pan sauce", - ), - }), - useTextExtract, - selector: selector, - }); - - await stagehand.close(); - - const { tablespoons_olive_oil, teaspoons_lemon_juice } = recipeDetails; - const expectedTablespoons = 2; - const expectedTeaspoons = 2; - - if ( - tablespoons_olive_oil !== expectedTablespoons || - teaspoons_lemon_juice !== expectedTeaspoons - ) { - const errors = []; - if (tablespoons_olive_oil !== expectedTablespoons) { - errors.push({ - message: - "Extracted tablespoons of olive oil do not match the extracted tablespoons of olive oil", - expected: expectedTablespoons.toString(), - actual: tablespoons_olive_oil.toString(), - }); - } - if (teaspoons_lemon_juice !== expectedTeaspoons) { - errors.push({ - message: - "Extracted teaspoons of lemon juice do not match the extracted teaspoons of lemon juice", - expected: expectedTeaspoons.toString(), - actual: teaspoons_lemon_juice.toString(), - }); - } - - logger.error({ - message: "Failed to extract correct recipe details", - level: 0, - auxiliary: { - errors: { - value: JSON.stringify(errors), - type: "object", - }, - }, - }); - - return { - _success: false, - error: "Recipe details extraction validation failed", - logs: logger.getLogs(), - debugUrl, - sessionUrl, - }; - } - - return { - _success: true, - recipeDetails: { - tablespoons_olive_oil: expectedTablespoons, - teaspoons_lemon_juice: expectedTeaspoons, - }, - logs: logger.getLogs(), - debugUrl, - sessionUrl, - }; -}; diff --git a/evals/tasks/extract_regulations_table.ts b/evals/tasks/extract_regulations_table.ts deleted file mode 100644 index 5bd75abe3..000000000 --- a/evals/tasks/extract_regulations_table.ts +++ /dev/null @@ -1,89 +0,0 @@ -import { EvalFunction } from "@/types/evals"; -import { z } from "zod"; - -export const extract_regulations_table: EvalFunction = async ({ - debugUrl, - sessionUrl, - stagehand, - logger, - useTextExtract, -}) => { - try { - await stagehand.page.goto( - "https://browserbase.github.io/stagehand-eval-sites/sites/ncc-numbering-plan/", - ); - - const xpath = - "/html/body/div[3]/main/div[2]/div[2]/div/div/div[2]/article/div[2]/div[1]/div/table"; - - const allottees = await stagehand.page.extract({ - instruction: - "Extract ALL of the Allottees and their corresponding name, area, and area code.", - schema: z.object({ - allottee_list: z.array( - z.object({ - allottee_name: z.string(), - area: z.string(), - area_code: z.string(), - access_code: z.string(), - }), - ), - }), - useTextExtract, - selector: xpath, - }); - - // Define the expected weather data - const allottees_expected_first = { - allottee_name: "101 Communications Limited", - area: "Lagos", - area_code: "0201", - access_code: "249", - }; - - const allottees_expected_last = { - allottee_name: "Airtel Networks Limited", - area: "National", - area_code: "0708", - access_code: "708", - }; - - const expected_length = 25; - - const allotteeList = allottees.allottee_list; - - // Check that the first entry, last entry, and total number match expectations - const isFirstCorrect = - JSON.stringify(allotteeList[0]) === - JSON.stringify(allottees_expected_first); - const isLastCorrect = - JSON.stringify(allotteeList[allotteeList.length - 1]) === - JSON.stringify(allottees_expected_last); - const isLengthCorrect = allotteeList.length === expected_length; - - const isRegulationsCorrect = - isFirstCorrect && isLastCorrect && isLengthCorrect; - - await stagehand.close(); - - return { - _success: isRegulationsCorrect, - regulationsData: allottees, - debugUrl, - sessionUrl, - logs: logger.getLogs(), - }; - } catch (error) { - console.error("Error or timeout occurred:", error); - - await stagehand.close(); - - return { - _success: false, - error: JSON.parse(JSON.stringify(error, null, 2)), - debugUrl, - sessionUrl, - logs: logger.getLogs(), - }; - } -}; diff --git a/evals/tasks/extract_repo_name.ts b/evals/tasks/extract_repo_name.ts deleted file mode 100644 index ff4f48238..000000000 --- a/evals/tasks/extract_repo_name.ts +++ /dev/null @@ -1,49 +0,0 @@ -import { EvalFunction } from "@/types/evals"; - -export const extract_repo_name: EvalFunction = async ({ - debugUrl, - sessionUrl, - stagehand, - logger, -}) => { - try { - await stagehand.page.goto("https://github.com/facebook/react"); - - const { extraction } = await stagehand.page.extract( - "extract the title of the Github repository. Do not include the owner of the repository.", - ); - - logger.log({ - message: "Extracted repo title", - level: 1, - auxiliary: { - repo_name: { - value: extraction, - type: "object", - }, - }, - }); - - await stagehand.close(); - - return { - _success: extraction === "react", - extraction, - debugUrl, - sessionUrl, - logs: logger.getLogs(), - }; - } catch (error) { - console.error("Error or timeout occurred:", error); - - await stagehand.close(); - - return { - _success: false, - error: JSON.parse(JSON.stringify(error, null, 2)), - debugUrl, - sessionUrl, - logs: logger.getLogs(), - }; - } -}; diff --git a/evals/tasks/extract_resistor_info.ts b/evals/tasks/extract_resistor_info.ts deleted file mode 100644 index 4f4f0b265..000000000 --- a/evals/tasks/extract_resistor_info.ts +++ /dev/null @@ -1,157 +0,0 @@ -import { EvalFunction } from "@/types/evals"; -import { normalizeString } from "@/evals/utils"; -import { z } from "zod"; - -export const extract_resistor_info: EvalFunction = async ({ - debugUrl, - sessionUrl, - stagehand, - logger, - useTextExtract, -}) => { - await stagehand.page.goto( - "https://browserbase.github.io/stagehand-eval-sites/sites/resistor/", - ); - - const result = await stagehand.page.extract({ - instruction: - "Extract the manufacturer standard lead time, tolerance percentage, resistance, and operating temperature range of the resistor.", - schema: z.object({ - manufacturer_standard_lead_time: z.string(), - tolerance_percentage: z.string(), - resistance: z.string(), - operating_temperature_range: z.string(), - }), - useTextExtract, - }); - - await stagehand.close(); - - const { - manufacturer_standard_lead_time, - tolerance_percentage, - resistance, - operating_temperature_range, - } = result; - - const expected = { - manufacturer_standard_lead_time: "11 Weeks", - tolerance_percentage: "±5", - resistance: "330 ohms", - operating_temperature_range: "-55°C ~ 155°C", - }; - - if ( - normalizeString(manufacturer_standard_lead_time) !== - normalizeString(expected.manufacturer_standard_lead_time) - ) { - logger.error({ - message: - "manufacturer standard lead time extracted does not match expected", - level: 0, - auxiliary: { - expected: { - value: normalizeString(expected.manufacturer_standard_lead_time), - type: "string", - }, - actual: { - value: normalizeString(manufacturer_standard_lead_time), - type: "string", - }, - }, - }); - return { - _success: false, - error: - "manufacturer standard lead time extracted does not match expected", - logs: logger.getLogs(), - debugUrl, - sessionUrl, - }; - } - - if ( - normalizeString(tolerance_percentage) !== - normalizeString(expected.tolerance_percentage) - ) { - logger.error({ - message: "Tolerance percentage extracted does not match expected", - level: 0, - auxiliary: { - expected: { - value: normalizeString(expected.tolerance_percentage), - type: "string", - }, - actual: { - value: normalizeString(tolerance_percentage), - type: "string", - }, - }, - }); - return { - _success: false, - error: "Tolerance percentage extracted does not match expected", - logs: logger.getLogs(), - debugUrl, - sessionUrl, - }; - } - - if (normalizeString(resistance) !== normalizeString(expected.resistance)) { - logger.error({ - message: "resistance extracted does not match expected", - level: 0, - auxiliary: { - expected: { - value: normalizeString(expected.resistance), - type: "string", - }, - actual: { - value: normalizeString(resistance), - type: "string", - }, - }, - }); - return { - _success: false, - error: "resistance extracted does not match expected", - logs: logger.getLogs(), - debugUrl, - sessionUrl, - }; - } - - if ( - normalizeString(operating_temperature_range) !== - normalizeString(expected.operating_temperature_range) - ) { - logger.error({ - message: "Operating temperature range extracted does not match expected", - level: 0, - auxiliary: { - expected: { - value: normalizeString(expected.operating_temperature_range), - type: "string", - }, - actual: { - value: normalizeString(operating_temperature_range), - type: "string", - }, - }, - }); - return { - _success: false, - error: "Operating temperature range extracted does not match expected", - logs: logger.getLogs(), - debugUrl, - sessionUrl, - }; - } - - return { - _success: true, - logs: logger.getLogs(), - debugUrl, - sessionUrl, - }; -}; diff --git a/evals/tasks/extract_rockauto.ts b/evals/tasks/extract_rockauto.ts deleted file mode 100644 index f154790de..000000000 --- a/evals/tasks/extract_rockauto.ts +++ /dev/null @@ -1,100 +0,0 @@ -import { EvalFunction } from "@/types/evals"; -import { z } from "zod"; - -export const extract_rockauto: EvalFunction = async ({ - debugUrl, - sessionUrl, - stagehand, - logger, - useTextExtract, -}) => { - await stagehand.page.goto( - "https://browserbase.github.io/stagehand-eval-sites/sites/rockauto/", - ); - await new Promise((resolve) => setTimeout(resolve, 5000)); - const result = await stagehand.page.extract({ - instruction: - "Extract the part number of all the coolant and antifreeze products in the 'economy' category. " + - "Do not include the manufacturer name. Do not include products from the premium category.", - schema: z.object({ - coolant_products: z.array( - z.object({ - part_number: z.string(), - }), - ), - }), - useTextExtract, - }); - - await stagehand.close(); - - const coolantProducts = result.coolant_products; - const expectedPartNumbers = [ - "GREEN5050GAL", - "719009", - "AF3300", - "AF3100", - "MV5050GAL", - ]; - const expectedLength = expectedPartNumbers.length; - - if (coolantProducts.length !== expectedLength) { - logger.error({ - message: "Incorrect number of coolant products extracted", - level: 0, - auxiliary: { - expected: { - value: expectedLength.toString(), - type: "integer", - }, - actual: { - value: coolantProducts.length.toString(), - type: "integer", - }, - }, - }); - return { - _success: false, - error: "Incorrect number of coolant products extracted", - logs: logger.getLogs(), - debugUrl, - sessionUrl, - }; - } - - const missingParts = expectedPartNumbers.filter( - (expectedPart) => - !coolantProducts.some((p) => p.part_number === expectedPart), - ); - - if (missingParts.length > 0) { - logger.error({ - message: "Missing expected part number(s)", - level: 0, - auxiliary: { - missingParts: { - value: JSON.stringify(missingParts), - type: "object", - }, - actualExtracted: { - value: JSON.stringify(coolantProducts), - type: "object", - }, - }, - }); - return { - _success: false, - error: `One or more expected part numbers were not found: ${missingParts.join(", ")}`, - logs: logger.getLogs(), - debugUrl, - sessionUrl, - }; - } - - return { - _success: true, - logs: logger.getLogs(), - debugUrl, - sessionUrl, - }; -}; diff --git a/evals/tasks/extract_single_link.ts b/evals/tasks/extract_single_link.ts deleted file mode 100644 index d2e19957f..000000000 --- a/evals/tasks/extract_single_link.ts +++ /dev/null @@ -1,52 +0,0 @@ -import { EvalFunction } from "@/types/evals"; -import { z } from "zod"; - -export const extract_single_link: EvalFunction = async ({ - logger, - debugUrl, - sessionUrl, - stagehand, -}) => { - try { - await stagehand.page.goto( - "https://browserbase.github.io/stagehand-eval-sites/sites/geniusee/", - ); - - const extraction = await stagehand.page.extract({ - instruction: "extract the link to the 'contact us' page", - schema: z.object({ - link: z.string().url(), - }), - }); - - await stagehand.close(); - const extractedLink = extraction.link; - const expectedLink = - "https://browserbase.github.io/stagehand-eval-sites/sites/geniusee/#contact"; - - if (extractedLink === expectedLink) { - return { - _success: true, - debugUrl, - sessionUrl, - logs: logger.getLogs(), - }; - } - return { - _success: false, - reason: `Extracted link: ${extractedLink} does not match expected link: ${expectedLink}`, - debugUrl, - sessionUrl, - logs: logger.getLogs(), - }; - } catch (error) { - await stagehand.close(); - return { - _success: false, - error: JSON.parse(JSON.stringify(error, null, 2)), - debugUrl, - sessionUrl, - logs: logger.getLogs(), - }; - } -}; diff --git a/evals/tasks/extract_snowshoeing_destinations.ts b/evals/tasks/extract_snowshoeing_destinations.ts deleted file mode 100644 index b35e7be64..000000000 --- a/evals/tasks/extract_snowshoeing_destinations.ts +++ /dev/null @@ -1,87 +0,0 @@ -import { z } from "zod"; -import { EvalFunction } from "@/types/evals"; - -export const extract_snowshoeing_destinations: EvalFunction = async ({ - debugUrl, - sessionUrl, - stagehand, - logger, - useTextExtract, -}) => { - try { - await stagehand.page.goto( - "https://www.cbisland.com/blog/10-snowshoeing-adventures-on-cape-breton-island/", - ); - - await stagehand.page.act({ action: "accept the cookies" }); - - const snowshoeing_regions = await stagehand.page.extract({ - instruction: - "Extract all the snowshoeing regions and the names of the trails within each region.", - schema: z.object({ - snowshoeing_regions: z.array( - z.object({ - region_name: z - .string() - .describe("The name of the snowshoeing region"), - trails: z - .array( - z.object({ - trail_name: z.string().describe("The name of the trail"), - }), - ) - .describe("The list of trails available in this region."), - }), - ), - }), - useTextExtract, - }); - - logger.log({ - message: "Extracted destinations and trails", - level: 1, - auxiliary: { - destinations: { - value: JSON.stringify(snowshoeing_regions), - type: "object", - }, - }, - }); - - await stagehand.close(); - - const _success = snowshoeing_regions.snowshoeing_regions.length === 10; - - return { - _success, - snowshoeing_regions, - debugUrl, - sessionUrl, - logs: logger.getLogs(), - }; - } catch (error) { - logger.error({ - message: "Error in extract_snowshoeing_destinations function", - level: 0, - auxiliary: { - error: { - value: error.message, - type: "string", - }, - trace: { - value: error.stack, - type: "string", - }, - }, - }); - return { - _success: false, - error: JSON.parse(JSON.stringify(error, null, 2)), - debugUrl, - sessionUrl, - logs: logger.getLogs(), - }; - } finally { - await stagehand.context.close().catch(() => {}); - } -}; diff --git a/evals/tasks/extract_staff_members.ts b/evals/tasks/extract_staff_members.ts deleted file mode 100644 index 7cf396773..000000000 --- a/evals/tasks/extract_staff_members.ts +++ /dev/null @@ -1,144 +0,0 @@ -import { z } from "zod"; -import { EvalFunction } from "@/types/evals"; - -export const extract_staff_members: EvalFunction = async ({ - debugUrl, - sessionUrl, - stagehand, - logger, - useTextExtract, -}) => { - await stagehand.page.goto( - "https://browserbase.github.io/stagehand-eval-sites/sites/panamcs/", - ); - - const result = await stagehand.page.extract({ - instruction: - "extract a list of ALL the staff members on this page, with their name and their job title", - schema: z.object({ - staff_members: z.array( - z.object({ - name: z.string(), - job_title: z.string(), - }), - ), - }), - useTextExtract, - }); - - const staff_members = result.staff_members; - await stagehand.close(); - - const expectedLength = 50; - - const expectedFirstItem = { - name: "Louis Alvarez", - job_title: "School Resource Officer", - }; - - const expectedLastItem = { - name: "Jessica Zipin", - job_title: "School Based Therapist", - }; - - if (staff_members.length !== expectedLength) { - logger.error({ - message: "Incorrect number of items extracted", - level: 0, - auxiliary: { - expected: { - value: expectedLength.toString(), - type: "integer", - }, - actual: { - value: staff_members.length.toString(), - type: "integer", - }, - }, - }); - - return { - _success: false, - error: "Incorrect number of staff members extracted", - logs: logger.getLogs(), - debugUrl, - sessionUrl, - }; - } - - // Check for the presence of the expected items - const firstItemExists = staff_members.some( - (member) => - member.name === expectedFirstItem.name && - member.job_title === expectedFirstItem.job_title, - ); - - if (!firstItemExists) { - logger.error({ - message: "Expected first staff member not found in extracted data", - level: 0, - auxiliary: { - expected: { - value: JSON.stringify(expectedFirstItem), - type: "object", - }, - actual: { - value: JSON.stringify(staff_members), - type: "object", - }, - }, - }); - - await stagehand.close(); - - return { - _success: false, - error: "Expected first staff member not found in extracted data", - logs: logger.getLogs(), - debugUrl, - sessionUrl, - }; - } - - const lastItemExists = staff_members.some( - (member) => - member.name === expectedLastItem.name && - member.job_title === expectedLastItem.job_title, - ); - - if (!lastItemExists) { - logger.error({ - message: "Expected last staff member not found in extracted data", - level: 0, - auxiliary: { - expected: { - value: JSON.stringify(expectedLastItem), - type: "object", - }, - actual: { - value: JSON.stringify(staff_members), - type: "object", - }, - }, - }); - - await stagehand.close(); - - return { - _success: false, - error: "Expected last staff member not found in extracted data", - logs: logger.getLogs(), - debugUrl, - sessionUrl, - }; - } - - await stagehand.close(); - - return { - _success: true, - logs: logger.getLogs(), - debugUrl, - sessionUrl, - }; -}; diff --git a/evals/tasks/extract_zillow.ts b/evals/tasks/extract_zillow.ts deleted file mode 100644 index 1902f281f..000000000 --- a/evals/tasks/extract_zillow.ts +++ /dev/null @@ -1,64 +0,0 @@ -import { z } from "zod"; -import { EvalFunction } from "../../types/evals"; - -export const extract_zillow: EvalFunction = async ({ - debugUrl, - sessionUrl, - stagehand, - logger, - useTextExtract, -}) => { - await stagehand.page.goto( - "https://browserbase.github.io/stagehand-eval-sites/sites/zillow/", - ); - // timeout for 5 seconds - await stagehand.page.waitForTimeout(5000); - const real_estate_listings = await stagehand.page.extract({ - instruction: - "Extract EACH AND EVERY HOME PRICE AND ADDRESS ON THE PAGE. DO NOT MISS ANY OF THEM.", - schema: z.object({ - listings: z.array( - z.object({ - price: z.string().describe("The price of the home"), - trails: z.string().describe("The address of the home"), - }), - ), - }), - useTextExtract, - }); - - await stagehand.close(); - const listings = real_estate_listings.listings; - const expectedLength = 38; - - if (listings.length < expectedLength) { - logger.error({ - message: "Incorrect number of listings extracted", - level: 0, - auxiliary: { - expected: { - value: expectedLength.toString(), - type: "integer", - }, - actual: { - value: listings.length.toString(), - type: "integer", - }, - }, - }); - return { - _success: false, - error: "Incorrect number of listings extracted", - logs: logger.getLogs(), - debugUrl, - sessionUrl, - }; - } - - return { - _success: true, - logs: logger.getLogs(), - debugUrl, - sessionUrl, - }; -}; diff --git a/evals/tasks/google_flights.ts b/evals/tasks/google_flights.ts deleted file mode 100644 index 73654dc1e..000000000 --- a/evals/tasks/google_flights.ts +++ /dev/null @@ -1,55 +0,0 @@ -import { EvalFunction } from "@/types/evals"; -import { ObserveResult } from "@/types/stagehand"; - -/** - * This eval attempts to click on an element that should not pass the playwright actionability check - * which happens by default if you call locator.click (more information here: - * https://playwright.dev/docs/actionability) - * - * If this eval passes, it means that we have correctly set {force: true} in performPlaywrightMethod, - * and the click was successful even though the target element (found by the xpath) did not - * pass the actionability check. - */ - -export const google_flights: EvalFunction = async ({ - debugUrl, - sessionUrl, - stagehand, - logger, -}) => { - await stagehand.page.goto( - "https://browserbase.github.io/stagehand-eval-sites/sites/google-flights/", - ); - - const observeResult: ObserveResult = { - selector: - "xpath=/html/body/c-wiz[2]/div/div[2]/c-wiz/div[1]/c-wiz/div[2]/div[2]/div[2]/div/div[2]/div[1]/ul/li[1]/div/div[1]", - description: "the first departing flight", - method: "click", - arguments: [], - }; - await stagehand.page.act(observeResult); - - const expectedUrl = - "https://browserbase.github.io/stagehand-eval-sites/sites/google-flights/return-flight.html"; - const currentUrl = stagehand.page.url(); - - await stagehand.close(); - - if (currentUrl === expectedUrl) { - return { - _success: true, - currentUrl, - debugUrl, - sessionUrl, - logs: logger.getLogs(), - }; - } - return { - _success: false, - error: "The current URL does not match expected.", - logs: logger.getLogs(), - debugUrl, - sessionUrl, - }; -}; diff --git a/evals/tasks/google_jobs.ts b/evals/tasks/google_jobs.ts deleted file mode 100644 index a8e139988..000000000 --- a/evals/tasks/google_jobs.ts +++ /dev/null @@ -1,96 +0,0 @@ -import { EvalFunction } from "@/types/evals"; -import { z } from "zod"; - -export const google_jobs: EvalFunction = async ({ - debugUrl, - sessionUrl, - stagehand, - logger, - useTextExtract, -}) => { - try { - await stagehand.page.goto("https://www.google.com/"); - await stagehand.page.act("click on the about page"); - await stagehand.page.act("click on the careers page"); - await stagehand.page.act("input data scientist into role"); - await stagehand.page.act("input new york city into location"); - await stagehand.page.act("click on the search button"); - await stagehand.page.act("click on the first job link"); - - const jobDetails = await stagehand.page.extract({ - instruction: - "Extract the following details from the job posting: application deadline, minimum qualifications (degree and years of experience), and preferred qualifications (degree and years of experience)", - schema: z.object({ - applicationDeadline: z - .string() - .describe("The date until which the application window will be open") - .nullable(), - minimumQualifications: z.object({ - degree: z.string().describe("The minimum required degree").nullable(), - yearsOfExperience: z - .number() - .describe("The minimum required years of experience") - .nullable(), - }), - preferredQualifications: z.object({ - degree: z.string().describe("The preferred degree").nullable(), - yearsOfExperience: z - .number() - .describe("The preferred years of experience") - .nullable(), - }), - }), - useTextExtract, - }); - - const isJobDetailsValid = - jobDetails && - Object.values(jobDetails).every( - (value) => - value !== null && - value !== undefined && - (typeof value !== "object" || - Object.values(value).every( - (v) => - v !== null && - v !== undefined && - (typeof v === "number" || typeof v === "string"), - )), - ); - - await stagehand.close(); - - return { - _success: isJobDetailsValid, - jobDetails, - debugUrl, - sessionUrl, - logs: logger.getLogs(), - }; - } catch (error) { - logger.error({ - message: "error in google_jobs function", - level: 0, - auxiliary: { - error: { - value: error.message, - type: "string", - }, - trace: { - value: error.stack, - type: "string", - }, - }, - }); - - await stagehand.close(); - - return { - _success: false, - debugUrl, - sessionUrl, - error: JSON.parse(JSON.stringify(error, null, 2)), - logs: logger.getLogs(), - }; - } -}; diff --git a/evals/tasks/history.ts b/evals/tasks/history.ts deleted file mode 100644 index a41eb256b..000000000 --- a/evals/tasks/history.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { EvalFunction } from "@/types/evals"; - -export const history: EvalFunction = async ({ - debugUrl, - sessionUrl, - stagehand, - logger, -}) => { - await stagehand.page.goto("https://docs.stagehand.dev"); - - await stagehand.page.act("click on the 'Quickstart' tab"); - - await stagehand.page.extract("Extract the title of the page"); - - await stagehand.page.observe("Find all links on the page"); - - const history = stagehand.history; - - const hasCorrectNumberOfEntries = history.length === 4; - - const hasNavigateEntry = history[0].method === "navigate"; - const hasActEntry = history[1].method === "act"; - const hasExtractEntry = history[2].method === "extract"; - const hasObserveEntry = history[3].method === "observe"; - - const allEntriesHaveTimestamps = history.every( - (entry) => - typeof entry.timestamp === "string" && entry.timestamp.length > 0, - ); - const allEntriesHaveResults = history.every( - (entry) => entry.result !== undefined, - ); - - await stagehand.close(); - - const success = - hasCorrectNumberOfEntries && - hasNavigateEntry && - hasActEntry && - hasExtractEntry && - hasObserveEntry && - allEntriesHaveTimestamps && - allEntriesHaveResults; - - return { - _success: success, - debugUrl, - sessionUrl, - logs: logger.getLogs(), - }; -}; diff --git a/evals/tasks/homedepot.ts b/evals/tasks/homedepot.ts deleted file mode 100644 index ca4b39b85..000000000 --- a/evals/tasks/homedepot.ts +++ /dev/null @@ -1,98 +0,0 @@ -import { EvalFunction } from "@/types/evals"; -import { z } from "zod"; - -export const homedepot: EvalFunction = async ({ - debugUrl, - sessionUrl, - stagehand, - logger, - useTextExtract, -}) => { - try { - await stagehand.page.goto("https://www.homedepot.com/"); - await stagehand.page.act("search for gas grills"); - await stagehand.page.act("click on the best selling gas grill"); - await stagehand.page.act("click on the Product Details"); - await stagehand.page.act("find the Primary Burner BTU"); - - const productSpecs = await stagehand.page.extract({ - instruction: "Extract the Primary exact Burner BTU of the product", - schema: z.object({ - productSpecs: z - .array( - z.object({ - burnerBTU: z.string().describe("Primary Burner BTU exact value"), - }), - ) - .describe("Gas grill Primary Burner BTU exact value"), - }), - useTextExtract, - }); - - logger.log({ - message: `gas grill primary burner BTU`, - level: 1, - auxiliary: { - productSpecs: { - value: JSON.stringify(productSpecs), - type: "object", - }, - }, - }); - - if ( - !productSpecs || - !productSpecs.productSpecs || - productSpecs.productSpecs.length !== 1 - ) { - await stagehand.close(); - - return { - _success: false, - productSpecs, - debugUrl, - sessionUrl, - logs: logger.getLogs(), - }; - } - - const hasFourZerosAndOne4 = - (productSpecs.productSpecs[0].burnerBTU.match(/0/g) || []).length === 4 && - (productSpecs.productSpecs[0].burnerBTU.match(/4/g) || []).length === 1; - - await stagehand.close(); - - return { - _success: hasFourZerosAndOne4, - productSpecs, - debugUrl, - sessionUrl, - logs: logger.getLogs(), - }; - } catch (error) { - logger.error({ - message: "error in homedepot function", - level: 0, - auxiliary: { - error: { - value: error.message, - type: "string", - }, - trace: { - value: error.stack, - type: "string", - }, - }, - }); - - await stagehand.close(); - - return { - _success: false, - error: JSON.parse(JSON.stringify(error, null, 2)), - debugUrl, - sessionUrl, - logs: logger.getLogs(), - }; - } -}; diff --git a/evals/tasks/imdb_movie_details.ts b/evals/tasks/imdb_movie_details.ts deleted file mode 100644 index 6e45bd2a0..000000000 --- a/evals/tasks/imdb_movie_details.ts +++ /dev/null @@ -1,99 +0,0 @@ -import { EvalFunction } from "@/types/evals"; -import { z } from "zod"; - -export const imdb_movie_details: EvalFunction = async ({ - debugUrl, - sessionUrl, - stagehand, - logger, - useTextExtract, -}) => { - await stagehand.page.goto("https://www.imdb.com/title/tt0111161/", { - waitUntil: "domcontentloaded", - }); - await stagehand.page.act({ - action: "click on the movie ratings", - }); - - const movieDetails = await stagehand.page.extract({ - instruction: "Extract the list of countries with the most ratings.", - schema: z.object({ - countries: z - .array(z.string()) - .describe("List of countries with the most ratings"), - }), - useTextExtract, - }); - - await stagehand.close(); - - const expectedCountries = [ - "United States", - "United Kingdom", - "Turkey", - "India", - "Germany", - ]; - - if (!movieDetails.countries || movieDetails.countries.length !== 5) { - logger.error({ - message: "Failed to extract exactly five countries", - level: 0, - auxiliary: { - expected: { - value: JSON.stringify(expectedCountries), - type: "object", - }, - actual: { - value: JSON.stringify(movieDetails.countries || []), - type: "object", - }, - }, - }); - - return { - _success: false, - error: "Incorrect number of countries extracted", - logs: logger.getLogs(), - debugUrl, - sessionUrl, - }; - } - - const missingCountries = expectedCountries.filter( - (country) => !movieDetails.countries.includes(country), - ); - - if (missingCountries.length > 0) { - logger.error({ - message: "Extracted countries do not match expected countries", - level: 0, - auxiliary: { - missing: { - value: JSON.stringify(missingCountries), - type: "object", - }, - extracted: { - value: JSON.stringify(movieDetails.countries), - type: "object", - }, - }, - }); - - return { - _success: false, - error: "Extracted countries do not match expected countries", - logs: logger.getLogs(), - debugUrl, - sessionUrl, - }; - } - - return { - _success: true, - countries: movieDetails.countries, - logs: logger.getLogs(), - debugUrl, - sessionUrl, - }; -}; diff --git a/evals/tasks/instructions.ts b/evals/tasks/instructions.ts deleted file mode 100644 index 75e4955b7..000000000 --- a/evals/tasks/instructions.ts +++ /dev/null @@ -1,44 +0,0 @@ -import { EvalFunction } from "@/types/evals"; - -export const instructions: EvalFunction = async ({ - debugUrl, - sessionUrl, - stagehand, - logger, -}) => { - try { - const page = stagehand.page; - - await page.goto("https://docs.browserbase.com/"); - - await page.act({ - action: "secret12345", - }); - - await page.waitForLoadState("domcontentloaded"); - - const url = page.url(); - - const isCorrectUrl = - url === "https://docs.browserbase.com/introduction/what-is-browserbase"; - - return { - _success: isCorrectUrl, - debugUrl, - sessionUrl, - logs: logger.getLogs(), - }; - } catch (error) { - console.error("Error or timeout occurred:", error); - - return { - _success: false, - error: JSON.parse(JSON.stringify(error, null, 2)), - debugUrl, - sessionUrl, - logs: logger.getLogs(), - }; - } finally { - await stagehand.close(); - } -}; diff --git a/evals/tasks/ionwave.ts b/evals/tasks/ionwave.ts deleted file mode 100644 index e4d4e200d..000000000 --- a/evals/tasks/ionwave.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { EvalFunction } from "@/types/evals"; - -export const ionwave: EvalFunction = async ({ - debugUrl, - sessionUrl, - stagehand, - logger, -}) => { - await stagehand.page.goto("https://elpasotexas.ionwave.net/Login.aspx"); - - await stagehand.page.act({ - action: 'Click on "Closed Bids"', - }); - - const expectedUrl = - "https://elpasotexas.ionwave.net/SourcingEvents.aspx?SourceType=2"; - const currentUrl = stagehand.page.url(); - - await stagehand.close(); - - return { - _success: currentUrl.startsWith(expectedUrl), - currentUrl, - debugUrl, - sessionUrl, - logs: logger.getLogs(), - }; -}; diff --git a/evals/tasks/ionwave_observe.ts b/evals/tasks/ionwave_observe.ts deleted file mode 100644 index 8fa3197dc..000000000 --- a/evals/tasks/ionwave_observe.ts +++ /dev/null @@ -1,62 +0,0 @@ -import { EvalFunction } from "@/types/evals"; - -export const ionwave_observe: EvalFunction = async ({ - debugUrl, - sessionUrl, - stagehand, - logger, -}) => { - await stagehand.page.goto("https://elpasotexas.ionwave.net/Login.aspx"); - - const observations = await stagehand.page.observe({ onlyVisible: true }); - - if (observations.length === 0) { - await stagehand.close(); - return { - _success: false, - observations, - debugUrl, - sessionUrl, - logs: logger.getLogs(), - }; - } - - const expectedLocator = `div.rowLinks:nth-child(27) > div:nth-child(1) > a:nth-child(1)`; - - const expectedResult = await stagehand.page - .locator(expectedLocator) - .first() - .innerText(); - - let foundMatch = false; - for (const observation of observations) { - try { - const observationResult = await stagehand.page - .locator(observation.selector) - .first() - .innerText(); - - if (observationResult === expectedResult) { - foundMatch = true; - break; - } - } catch (error) { - console.warn( - `Failed to check observation with selector ${observation.selector}:`, - error.message, - ); - continue; - } - } - - await stagehand.close(); - - return { - _success: foundMatch, - expected: expectedResult, - observations, - debugUrl, - sessionUrl, - logs: logger.getLogs(), - }; -}; diff --git a/evals/tasks/nextChunk.ts b/evals/tasks/nextChunk.ts deleted file mode 100644 index 6591cb8c5..000000000 --- a/evals/tasks/nextChunk.ts +++ /dev/null @@ -1,75 +0,0 @@ -import { Stagehand } from "@/dist"; -import { EvalFunction } from "@/types/evals"; - -export const nextChunk: EvalFunction = async ({ - logger, - stagehandConfig, - debugUrl, - sessionUrl, -}) => { - const stagehand = new Stagehand({ - ...stagehandConfig, - domSettleTimeoutMs: 3000, - }); - await stagehand.init(); - - await stagehand.page.goto("https://www.apartments.com/san-francisco-ca/"); - await stagehand.page.act({ - action: "click on the all filters button", - }); - - const { initialScrollTop, chunkHeight } = await stagehand.page.evaluate( - () => { - const container = document.querySelector( - "#advancedFilters > div", - ) as HTMLElement; - if (!container) { - console.warn( - "Could not find #advancedFilters > div. Returning 0 for measurements.", - ); - return { initialScrollTop: 0, chunkHeight: 0 }; - } - return { - initialScrollTop: container.scrollTop, - chunkHeight: container.getBoundingClientRect().height, - }; - }, - ); - - await stagehand.page.act({ - action: "scroll down one chunk on the filters modal", - }); - - await new Promise((resolve) => setTimeout(resolve, 2000)); - - const newScrollTop = await stagehand.page.evaluate(() => { - const container = document.querySelector( - "#advancedFilters > div", - ) as HTMLElement; - return container?.scrollTop ?? 0; - }); - - await stagehand.close(); - - const actualDiff = newScrollTop - initialScrollTop; - const threshold = 20; // allowable difference in px - const scrolledOneChunk = Math.abs(actualDiff - chunkHeight) <= threshold; - - const evaluationResult = scrolledOneChunk - ? { - _success: true, - logs: logger.getLogs(), - debugUrl, - sessionUrl, - message: `Successfully scrolled ~one chunk: expected ~${chunkHeight}, got ${actualDiff}`, - } - : { - _success: false, - logs: logger.getLogs(), - debugUrl, - sessionUrl, - message: `Scroll difference expected ~${chunkHeight} but only scrolled ${actualDiff}.`, - }; - - return evaluationResult; -}; diff --git a/evals/tasks/nonsense_action.ts b/evals/tasks/nonsense_action.ts deleted file mode 100644 index be6f90dbd..000000000 --- a/evals/tasks/nonsense_action.ts +++ /dev/null @@ -1,34 +0,0 @@ -import { EvalFunction } from "@/types/evals"; - -export const nonsense_action: EvalFunction = async ({ - debugUrl, - sessionUrl, - stagehand, - logger, -}) => { - try { - await stagehand.page.goto("https://www.homedepot.com/"); - - const result = await stagehand.page.act({ - action: "what is the capital of the moon?", - }); - - return { - _success: !result.success, // We expect this to fail - debugUrl, - sessionUrl, - logs: logger.getLogs(), - }; - } catch (error) { - console.error(`Error in nonsense_action function: ${error.message}`); - return { - _success: false, - error: JSON.parse(JSON.stringify(error, null, 2)), - debugUrl, - sessionUrl, - logs: logger.getLogs(), - }; - } finally { - await stagehand.close(); - } -}; diff --git a/evals/tasks/observe_amazon_add_to_cart.ts b/evals/tasks/observe_amazon_add_to_cart.ts deleted file mode 100644 index a9b810380..000000000 --- a/evals/tasks/observe_amazon_add_to_cart.ts +++ /dev/null @@ -1,55 +0,0 @@ -import { EvalFunction } from "@/types/evals"; - -export const observe_amazon_add_to_cart: EvalFunction = async ({ - debugUrl, - sessionUrl, - stagehand, - logger, -}) => { - await stagehand.page.goto( - "https://browserbase.github.io/stagehand-eval-sites/sites/amazon/", - ); - - await stagehand.page.waitForTimeout(5000); - - const observations1 = await stagehand.page.observe({ - instruction: "Find and click the 'Add to Cart' button", - onlyVisible: false, - returnAction: true, - }); - - console.log(observations1); - - // Example of using performPlaywrightMethod if you have the xpath - if (observations1.length > 0) { - const action1 = observations1[0]; - await stagehand.page.act(action1); - } - - await stagehand.page.waitForTimeout(2000); - - const observations2 = await stagehand.page.observe({ - instruction: "Find and click the 'Proceed to checkout' button", - }); - - // Example of using performPlaywrightMethod if you have the xpath - if (observations2.length > 0) { - const action2 = observations2[0]; - await stagehand.page.act(action2); - } - await stagehand.page.waitForTimeout(2000); - - const currentUrl = stagehand.page.url(); - const expectedUrlPrefix = - "https://browserbase.github.io/stagehand-eval-sites/sites/amazon/sign-in.html"; - - await stagehand.close(); - - return { - _success: currentUrl.startsWith(expectedUrlPrefix), - currentUrl, - debugUrl, - sessionUrl, - logs: logger.getLogs(), - }; -}; diff --git a/evals/tasks/observe_github.ts b/evals/tasks/observe_github.ts deleted file mode 100644 index 036255f85..000000000 --- a/evals/tasks/observe_github.ts +++ /dev/null @@ -1,88 +0,0 @@ -import { EvalFunction } from "@/types/evals"; - -export const observe_github: EvalFunction = async ({ - debugUrl, - sessionUrl, - stagehand, - logger, -}) => { - await stagehand.page.goto("https://github.com/numpy/numpy/tree/main/numpy"); - - const observations = await stagehand.page.observe({ - instruction: "find the scrollable element that holds the repos file tree.", - }); - - if (observations.length === 0) { - await stagehand.close(); - return { - _success: false, - observations, - debugUrl, - sessionUrl, - logs: logger.getLogs(), - }; - } - - const possibleLocators = [ - `#repos-file-tree > div.Box-sc-g0xbh4-0.jbQqON > div > div > div > nav > ul`, - `#repos-file-tree > div.Box-sc-g0xbh4-0.jbQqON > div > div > div > nav`, - `#repos-file-tree > div.Box-sc-g0xbh4-0.jbQqON`, - ]; - - const possibleHandles = []; - for (const locatorStr of possibleLocators) { - const locator = stagehand.page.locator(locatorStr); - const handle = await locator.elementHandle(); - if (handle) { - possibleHandles.push({ locatorStr, handle }); - } - } - - let foundMatch = false; - let matchedLocator: string | null = null; - - for (const observation of observations) { - try { - const observationLocator = stagehand.page - .locator(observation.selector) - .first(); - const observationHandle = await observationLocator.elementHandle(); - if (!observationHandle) { - continue; - } - - for (const { locatorStr, handle: candidateHandle } of possibleHandles) { - const isSameNode = await observationHandle.evaluate( - (node, otherNode) => node === otherNode, - candidateHandle, - ); - if (isSameNode) { - foundMatch = true; - matchedLocator = locatorStr; - break; - } - } - - if (foundMatch) { - break; - } - } catch (error) { - console.warn( - `Failed to check observation with selector ${observation.selector}:`, - error.message, - ); - continue; - } - } - - await stagehand.close(); - - return { - _success: foundMatch, - matchedLocator, - observations, - debugUrl, - sessionUrl, - logs: logger.getLogs(), - }; -}; diff --git a/evals/tasks/observe_iframes1.ts b/evals/tasks/observe_iframes1.ts deleted file mode 100644 index f1cbc68ee..000000000 --- a/evals/tasks/observe_iframes1.ts +++ /dev/null @@ -1,87 +0,0 @@ -import { EvalFunction } from "@/types/evals"; - -export const observe_iframes1: EvalFunction = async ({ - debugUrl, - sessionUrl, - stagehand, - logger, -}) => { - await stagehand.page.goto("https://tucowsdomains.com/abuse-form/phishing/"); - - const observations = await stagehand.page.observe({ - instruction: "find the main header of the page", - }); - - if (observations.length === 0) { - await stagehand.close(); - return { - _success: false, - observations, - debugUrl, - sessionUrl, - logs: logger.getLogs(), - }; - } - - const possibleLocators = [ - `#primary > div.singlePage > section > div > div > article > div > iframe`, - `#primary > div.heroBanner > section > div > h1`, - ]; - - const possibleHandles = []; - for (const locatorStr of possibleLocators) { - const locator = stagehand.page.locator(locatorStr); - const handle = await locator.elementHandle(); - if (handle) { - possibleHandles.push({ locatorStr, handle }); - } - } - - let foundMatch = false; - let matchedLocator: string | null = null; - - for (const observation of observations) { - try { - const observationLocator = stagehand.page - .locator(observation.selector) - .first(); - const observationHandle = await observationLocator.elementHandle(); - if (!observationHandle) { - continue; - } - - for (const { locatorStr, handle: candidateHandle } of possibleHandles) { - const isSameNode = await observationHandle.evaluate( - (node, otherNode) => node === otherNode, - candidateHandle, - ); - if (isSameNode) { - foundMatch = true; - matchedLocator = locatorStr; - break; - } - } - - if (foundMatch) { - break; - } - } catch (error) { - console.warn( - `Failed to check observation with selector ${observation.selector}:`, - error.message, - ); - continue; - } - } - - await stagehand.close(); - - return { - _success: foundMatch, - matchedLocator, - observations, - debugUrl, - sessionUrl, - logs: logger.getLogs(), - }; -}; diff --git a/evals/tasks/observe_iframes2.ts b/evals/tasks/observe_iframes2.ts deleted file mode 100644 index e02ebc7ab..000000000 --- a/evals/tasks/observe_iframes2.ts +++ /dev/null @@ -1,106 +0,0 @@ -import { Stagehand } from "@/dist"; -import { EvalFunction } from "@/types/evals"; -import { ObserveResult } from "@/types/stagehand"; - -export const observe_iframes2: EvalFunction = async ({ - logger, - stagehandConfig, - debugUrl, - sessionUrl, -}) => { - const stagehand = new Stagehand({ - ...stagehandConfig, - }); - await stagehand.init(); - - await stagehand.page.goto( - "https://iframetester.com/?url=https://shopify.com", - ); - await new Promise((resolve) => setTimeout(resolve, 5000)); - - let observations: ObserveResult[]; - try { - observations = await stagehand.page.observe({ - instruction: "find the main header of the page", - }); - } catch (err) { - await stagehand.close(); - return { - _success: false, - message: err.message, - debugUrl, - sessionUrl, - logs: logger.getLogs(), - }; - } - - if (observations.length === 0) { - await stagehand.close(); - return { - _success: false, - observations, - debugUrl, - sessionUrl, - logs: logger.getLogs(), - }; - } - - const possibleLocators = [`#iframe-window`, `body > header > h1`]; - - const possibleHandles = []; - for (const locatorStr of possibleLocators) { - const locator = stagehand.page.locator(locatorStr); - const handle = await locator.elementHandle(); - if (handle) { - possibleHandles.push({ locatorStr, handle }); - } - } - - let foundMatch = false; - let matchedLocator: string | null = null; - - for (const observation of observations) { - try { - const observationLocator = stagehand.page - .locator(observation.selector) - .first(); - const observationHandle = await observationLocator.elementHandle(); - if (!observationHandle) { - continue; - } - - for (const { locatorStr, handle: candidateHandle } of possibleHandles) { - const isSameNode = await observationHandle.evaluate( - (node, otherNode) => node === otherNode, - candidateHandle, - ); - if (isSameNode) { - foundMatch = true; - matchedLocator = locatorStr; - break; - } - } - - if (foundMatch) { - break; - } - } catch (error) { - console.warn( - `Failed to check observation with selector ${observation.selector}:`, - error.message, - ); - continue; - } - } - - await stagehand.close(); - - return { - _success: foundMatch, - matchedLocator, - observations, - debugUrl, - sessionUrl, - logs: logger.getLogs(), - }; -}; diff --git a/evals/tasks/observe_simple_google_search.ts b/evals/tasks/observe_simple_google_search.ts deleted file mode 100644 index 7e6fd1331..000000000 --- a/evals/tasks/observe_simple_google_search.ts +++ /dev/null @@ -1,61 +0,0 @@ -import { EvalFunction } from "@/types/evals"; -import { performPlaywrightMethod } from "@/lib/a11y/utils"; - -export const observe_simple_google_search: EvalFunction = async ({ - debugUrl, - sessionUrl, - stagehand, - logger, -}) => { - await stagehand.page.goto("https://www.google.com"); - - const observation1 = await stagehand.page.observe({ - instruction: "Find the search bar and enter 'OpenAI'", - onlyVisible: false, - returnAction: true, - }); - console.log(observation1); - - if (observation1.length > 0) { - const action1 = observation1[0]; - await performPlaywrightMethod( - stagehand.page, - stagehand.logger, - action1.method, - action1.arguments, - action1.selector.replace("xpath=", ""), - ); - } - await stagehand.page.waitForTimeout(5000); - const observation2 = await stagehand.page.observe({ - instruction: "Click the search button in the suggestions dropdown", - onlyVisible: false, - returnAction: true, - }); - console.log(observation2); - - if (observation2.length > 0) { - const action2 = observation2[0]; - await performPlaywrightMethod( - stagehand.page, - stagehand.logger, - action2.method, - action2.arguments, - action2.selector.replace("xpath=", ""), - ); - } - await stagehand.page.waitForTimeout(5000); - - const expectedUrl = "https://www.google.com/search?q=OpenAI"; - const currentUrl = stagehand.page.url(); - - await stagehand.close(); - - return { - _success: currentUrl.startsWith(expectedUrl), - currentUrl, - debugUrl, - sessionUrl, - logs: logger.getLogs(), - }; -}; diff --git a/evals/tasks/observe_taxes.ts b/evals/tasks/observe_taxes.ts deleted file mode 100644 index 3ba0b3676..000000000 --- a/evals/tasks/observe_taxes.ts +++ /dev/null @@ -1,73 +0,0 @@ -import { EvalFunction } from "@/types/evals"; - -export const observe_taxes: EvalFunction = async ({ - debugUrl, - sessionUrl, - stagehand, - logger, -}) => { - await stagehand.page.goto("https://file.1040.com/estimate/"); - - const observations = await stagehand.page.observe({ - instruction: "Find all the form input elements under the 'Income' section", - }); - - if (observations.length === 0) { - await stagehand.close(); - return { - _success: false, - observations, - debugUrl, - sessionUrl, - logs: logger.getLogs(), - }; - } else if (observations.length < 13) { - await stagehand.close(); - return { - _success: false, - observations, - debugUrl, - sessionUrl, - logs: logger.getLogs(), - }; - } - - const expectedLocator = `#tpWages`; - - const expectedResult = await stagehand.page - .locator(expectedLocator) - .first() - .innerText(); - - let foundMatch = false; - for (const observation of observations) { - try { - const observationResult = await stagehand.page - .locator(observation.selector) - .first() - .innerText(); - - if (observationResult === expectedResult) { - foundMatch = true; - break; - } - } catch (error) { - console.warn( - `Failed to check observation with selector ${observation.selector}:`, - error.message, - ); - continue; - } - } - - await stagehand.close(); - - return { - _success: foundMatch, - expected: expectedResult, - observations, - debugUrl, - sessionUrl, - logs: logger.getLogs(), - }; -}; diff --git a/evals/tasks/observe_vantechjournal.ts b/evals/tasks/observe_vantechjournal.ts deleted file mode 100644 index c3eef6a71..000000000 --- a/evals/tasks/observe_vantechjournal.ts +++ /dev/null @@ -1,74 +0,0 @@ -import { EvalFunction } from "@/types/evals"; - -export const observe_vantechjournal: EvalFunction = async ({ - debugUrl, - sessionUrl, - stagehand, - logger, -}) => { - await stagehand.page.goto("https://vantechjournal.com/archive?page=8"); - await stagehand.page.waitForTimeout(1000); - - const observations = await stagehand.page.observe({ - instruction: "find the button that takes us to the 11th page", - }); - - if (observations.length === 0) { - await stagehand.close(); - return { - _success: false, - observations, - debugUrl, - sessionUrl, - logs: logger.getLogs(), - }; - } - - const expectedLocator = `a.rounded-lg:nth-child(8)`; - - const expectedResult = await stagehand.page.locator(expectedLocator); - - let foundMatch = false; - - for (const observation of observations) { - try { - const observationLocator = stagehand.page - .locator(observation.selector) - .first(); - const observationHandle = await observationLocator.elementHandle(); - const expectedHandle = await expectedResult.elementHandle(); - - if (!observationHandle || !expectedHandle) { - // Couldn’t get handles, skip - continue; - } - - const isSameNode = await observationHandle.evaluate( - (node, otherNode) => node === otherNode, - expectedHandle, - ); - - if (isSameNode) { - foundMatch = true; - break; - } - } catch (error) { - console.warn( - `Failed to check observation with selector ${observation.selector}:`, - error.message, - ); - continue; - } - } - - await stagehand.close(); - - return { - _success: foundMatch, - expected: expectedResult, - observations, - debugUrl, - sessionUrl, - logs: logger.getLogs(), - }; -}; diff --git a/evals/tasks/observe_yc_startup.ts b/evals/tasks/observe_yc_startup.ts deleted file mode 100644 index bdc654da5..000000000 --- a/evals/tasks/observe_yc_startup.ts +++ /dev/null @@ -1,89 +0,0 @@ -import { EvalFunction } from "@/types/evals"; - -export const observe_yc_startup: EvalFunction = async ({ - debugUrl, - sessionUrl, - stagehand, - logger, -}) => { - await stagehand.page.goto("https://www.ycombinator.com/companies"); - await stagehand.page.waitForLoadState("networkidle"); - - const observations = await stagehand.page.observe({ - instruction: - "Click the container element that holds links to each of the startup companies. The companies each have a name, a description, and a link to their website.", - }); - - if (observations.length === 0) { - await stagehand.close(); - return { - _success: false, - observations, - debugUrl, - sessionUrl, - logs: logger.getLogs(), - }; - } - - const possibleLocators = [ - `div._rightCol_i9oky_592`, - `div._section_i9oky_163._results_i9oky_343`, - ]; - - const possibleHandles = []; - for (const locatorStr of possibleLocators) { - const locator = stagehand.page.locator(locatorStr); - const handle = await locator.elementHandle(); - if (handle) { - possibleHandles.push({ locatorStr, handle }); - } - } - - let foundMatch = false; - let matchedLocator: string | null = null; - - for (const observation of observations) { - try { - const observationLocator = stagehand.page - .locator(observation.selector) - .first(); - const observationHandle = await observationLocator.elementHandle(); - if (!observationHandle) { - continue; - } - - for (const { locatorStr, handle: candidateHandle } of possibleHandles) { - const isSameNode = await observationHandle.evaluate( - (node, otherNode) => node === otherNode, - candidateHandle, - ); - if (isSameNode) { - foundMatch = true; - matchedLocator = locatorStr; - break; - } - } - - if (foundMatch) { - break; - } - } catch (error) { - console.warn( - `Failed to check observation with selector ${observation.selector}:`, - error.message, - ); - continue; - } - } - - await stagehand.close(); - - return { - _success: foundMatch, - matchedLocator, - observations, - debugUrl, - sessionUrl, - logs: logger.getLogs(), - }; -}; diff --git a/evals/tasks/panamcs.ts b/evals/tasks/panamcs.ts deleted file mode 100644 index 2b68d096e..000000000 --- a/evals/tasks/panamcs.ts +++ /dev/null @@ -1,66 +0,0 @@ -import { EvalFunction } from "@/types/evals"; - -export const panamcs: EvalFunction = async ({ - debugUrl, - sessionUrl, - stagehand, - logger, -}) => { - await stagehand.page.goto( - "https://browserbase.github.io/stagehand-eval-sites/sites/panamcs/", - ); - - const observations = await stagehand.page.observe( - "click the 'about us' link", - ); - - if (observations.length === 0) { - await stagehand.close(); - return { - _success: false, - observations, - debugUrl, - sessionUrl, - logs: logger.getLogs(), - }; - } - - const expectedLocator = `#menu > li:nth-child(1) > a`; - - const expectedResult = await stagehand.page - .locator(expectedLocator) - .first() - .innerText(); - - let foundMatch = false; - for (const observation of observations) { - try { - const observationResult = await stagehand.page - .locator(observation.selector) - .first() - .innerText(); - - if (observationResult === expectedResult) { - foundMatch = true; - break; - } - } catch (error) { - console.warn( - `Failed to check observation with selector ${observation.selector}:`, - error.message, - ); - continue; - } - } - - await stagehand.close(); - - return { - _success: foundMatch, - expected: expectedResult, - observations, - debugUrl, - sessionUrl, - logs: logger.getLogs(), - }; -}; diff --git a/evals/tasks/peeler_complex.ts b/evals/tasks/peeler_complex.ts deleted file mode 100644 index 4b9ad170c..000000000 --- a/evals/tasks/peeler_complex.ts +++ /dev/null @@ -1,68 +0,0 @@ -import { EvalFunction } from "@/types/evals"; -import { z } from "zod"; - -export const peeler_complex: EvalFunction = async ({ - debugUrl, - sessionUrl, - stagehand, - logger, - useTextExtract, -}) => { - try { - await stagehand.page.goto(`https://chefstoys.com/`, { timeout: 60000 }); - await stagehand.page.waitForLoadState("networkidle"); - - await stagehand.page.act("find the button to close the popup"); - await stagehand.page.act({ - action: "search for %search_query%", - variables: { - search_query: "peeler", - }, - }); - - await stagehand.page.act({ - action: 'click on the first "OXO" brand peeler', - }); - - const { price } = await stagehand.page.extract({ - instruction: "get the price of the peeler", - schema: z.object({ price: z.number().nullable() }), - useTextExtract, - }); - - await stagehand.close(); - - return { - _success: price === 11.99, - price, - debugUrl, - sessionUrl, - logs: logger.getLogs(), - }; - } catch (error) { - logger.error({ - message: "error in peeler_complex function", - level: 0, - auxiliary: { - error: { - value: JSON.stringify(error, null, 2), - type: "object", - }, - trace: { - value: error.stack, - type: "string", - }, - }, - }); - - await stagehand.close(); - - return { - _success: false, - error: JSON.parse(JSON.stringify(error, null, 2)), - debugUrl, - sessionUrl, - logs: logger.getLogs(), - }; - } -}; diff --git a/evals/tasks/peeler_simple.ts b/evals/tasks/peeler_simple.ts deleted file mode 100644 index 12628c6fa..000000000 --- a/evals/tasks/peeler_simple.ts +++ /dev/null @@ -1,39 +0,0 @@ -import { EvalFunction } from "@/types/evals"; -import { StagehandEnvironmentError } from "@/types/stagehandErrors"; - -const env: "BROWSERBASE" | "LOCAL" = - process.env.EVAL_ENV?.toLowerCase() === "browserbase" - ? "BROWSERBASE" - : "LOCAL"; - -export const peeler_simple: EvalFunction = async ({ - debugUrl, - sessionUrl, - stagehand, - logger, -}) => { - if (env === "BROWSERBASE") { - throw new StagehandEnvironmentError( - "BROWSERBASE", - "LOCAL", - "peeler_simple eval", - ); - } - - await stagehand.page.goto(`file://${process.cwd()}/evals/assets/peeler.html`); - await stagehand.page.act({ action: "add the peeler to cart" }); - - const successMessageLocator = stagehand.page.locator( - 'text="Congratulations, you have 1 A in your cart"', - ); - const isVisible = await successMessageLocator.isVisible(); - - await stagehand.close(); - - return { - _success: isVisible, - debugUrl, - sessionUrl, - logs: logger.getLogs(), - }; -}; diff --git a/evals/tasks/prevChunk.ts b/evals/tasks/prevChunk.ts deleted file mode 100644 index 0db2266bf..000000000 --- a/evals/tasks/prevChunk.ts +++ /dev/null @@ -1,70 +0,0 @@ -import { Stagehand } from "@/dist"; -import { EvalFunction } from "@/types/evals"; - -export const prevChunk: EvalFunction = async ({ - logger, - stagehandConfig, - debugUrl, - sessionUrl, -}) => { - const stagehand = new Stagehand({ - ...stagehandConfig, - domSettleTimeoutMs: 3000, - }); - await stagehand.init(); - - await stagehand.page.goto( - "https://browserbase.github.io/stagehand-eval-sites/sites/aigrant/", - ); - await new Promise((resolve) => setTimeout(resolve, 2000)); - const { initialScrollTop, chunkHeight } = await stagehand.page.evaluate( - () => { - const halfPage = document.body.scrollHeight / 2; - - window.scrollTo({ - top: halfPage, - left: 0, - behavior: "instant", - }); - - const chunk = window.innerHeight; - - return { - initialScrollTop: window.scrollY, - chunkHeight: chunk, - }; - }, - ); - await new Promise((resolve) => setTimeout(resolve, 2000)); - await stagehand.page.act({ - action: "scroll up one chunk", - }); - - await new Promise((resolve) => setTimeout(resolve, 5000)); - - const finalScrollTop = await stagehand.page.evaluate(() => window.scrollY); - - await stagehand.close(); - - const actualDiff = initialScrollTop - finalScrollTop; - const threshold = 20; // px tolerance - const scrolledOneChunk = Math.abs(actualDiff - chunkHeight) <= threshold; - - const evaluationResult = scrolledOneChunk - ? { - _success: true, - logs: logger.getLogs(), - debugUrl, - sessionUrl, - message: `Successfully scrolled ~one chunk UP: expected ~${chunkHeight}, got ${actualDiff}.`, - } - : { - _success: false, - logs: logger.getLogs(), - debugUrl, - sessionUrl, - message: `Scroll difference expected ~${chunkHeight} but only scrolled ${actualDiff}.`, - }; - - return evaluationResult; -}; diff --git a/evals/tasks/rakuten_jp.ts b/evals/tasks/rakuten_jp.ts deleted file mode 100644 index cf8ea55fe..000000000 --- a/evals/tasks/rakuten_jp.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { EvalFunction } from "@/types/evals"; - -export const rakuten_jp: EvalFunction = async ({ - debugUrl, - sessionUrl, - stagehand, - logger, -}) => { - await stagehand.page.goto("https://www.rakuten.co.jp/"); - await stagehand.page.act({ action: "click on online supermarket" }); - - await stagehand.page.act({ action: "if there is a popup, close it" }); - - await stagehand.page.act({ - action: "navigate to Inageya Online Supermarket", - }); - await stagehand.page.act({ action: "click the search bar input" }); - await stagehand.page.act({ action: "search for '香菜'" }); - - const url = stagehand.page.url(); - const successUrl = - "https://netsuper.rakuten.co.jp/inageya/search/?keyword=%E9%A6%99%E8%8F%9C"; - - await stagehand.close(); - - return { - _success: url === successUrl, - debugUrl, - sessionUrl, - logs: logger.getLogs(), - }; -}; diff --git a/evals/tasks/sciquest.ts b/evals/tasks/sciquest.ts deleted file mode 100644 index 42bc535d7..000000000 --- a/evals/tasks/sciquest.ts +++ /dev/null @@ -1,71 +0,0 @@ -import { EvalFunction } from "@/types/evals"; -import { z } from "zod"; - -export const sciquest: EvalFunction = async ({ - debugUrl, - sessionUrl, - stagehand, - logger, - useTextExtract, -}) => { - await stagehand.page.goto( - "https://bids.sciquest.com/apps/Router/PublicEvent?tab=PHX_NAV_SourcingAllOpps&CustomerOrg=StateOfUtah", - ); - - await stagehand.page.act({ - action: 'Click on the "Closed" tab', - }); - - const result = await stagehand.page.extract({ - instruction: - "Extract the total number of results that the search produced. Not the number of results displayed on the page.", - schema: z.object({ - total_results: z.string(), - }), - useTextExtract, - }); - - await stagehand.close(); - - const { total_results } = result; - - const expectedNumber = 12637; - const extractedNumber = parseInt(total_results.replace(/[^\d]/g, ""), 10); - - const isWithinRange = - extractedNumber >= expectedNumber - 1000 && - extractedNumber <= expectedNumber + 1000; - - if (!isWithinRange) { - logger.error({ - message: "Total number of results is not within the expected range", - level: 0, - auxiliary: { - expected: { - value: `${expectedNumber} ± 1000`, - type: "string", - }, - actual: { - value: extractedNumber.toString(), - type: "integer", - }, - }, - }); - return { - _success: false, - error: "Total number of results is not within the expected range", - extractedNumber, - debugUrl, - sessionUrl, - logs: logger.getLogs(), - }; - } - - return { - _success: true, - extractedNumber, - debugUrl, - sessionUrl, - logs: logger.getLogs(), - }; -}; diff --git a/evals/tasks/scroll_50.ts b/evals/tasks/scroll_50.ts deleted file mode 100644 index c6abb53c3..000000000 --- a/evals/tasks/scroll_50.ts +++ /dev/null @@ -1,46 +0,0 @@ -import { EvalFunction } from "@/types/evals"; - -export const scroll_50: EvalFunction = async ({ - debugUrl, - sessionUrl, - stagehand, - logger, -}) => { - await stagehand.page.goto( - "https://browserbase.github.io/stagehand-eval-sites/sites/aigrant/", - ); - await stagehand.page.act({ - action: "Scroll 50% down the page", - }); - - await new Promise((resolve) => setTimeout(resolve, 5000)); - - // Get the current scroll position and total scroll height - const scrollInfo = await stagehand.page.evaluate(() => { - return { - scrollTop: window.scrollY + window.innerHeight / 2, - scrollHeight: document.documentElement.scrollHeight, - }; - }); - - await stagehand.close(); - - const halfwayScroll = scrollInfo.scrollHeight / 2; - const halfwayReached = Math.abs(scrollInfo.scrollTop - halfwayScroll) <= 200; - const evaluationResult = halfwayReached - ? { - _success: true, - logs: logger.getLogs(), - debugUrl, - sessionUrl, - } - : { - _success: false, - logs: logger.getLogs(), - debugUrl, - sessionUrl, - message: `Scroll position (${scrollInfo.scrollTop}px) is not halfway down the page (${halfwayScroll}px).`, - }; - - return evaluationResult; -}; diff --git a/evals/tasks/scroll_75.ts b/evals/tasks/scroll_75.ts deleted file mode 100644 index 3959837ca..000000000 --- a/evals/tasks/scroll_75.ts +++ /dev/null @@ -1,54 +0,0 @@ -import { Stagehand } from "@/dist"; -import { EvalFunction } from "@/types/evals"; - -export const scroll_75: EvalFunction = async ({ - logger, - stagehandConfig, - debugUrl, - sessionUrl, -}) => { - const stagehand = new Stagehand({ - ...stagehandConfig, - domSettleTimeoutMs: 3000, - }); - await stagehand.init(); - - await stagehand.page.goto( - "https://browserbase.github.io/stagehand-eval-sites/sites/aigrant/", - ); - await stagehand.page.act({ - action: "Scroll 75% down the page", - }); - - await new Promise((resolve) => setTimeout(resolve, 5000)); - - // Get the current scroll position and total scroll height - const scrollInfo = await stagehand.page.evaluate(() => { - return { - scrollTop: window.scrollY + window.innerHeight * 0.75, - scrollHeight: document.documentElement.scrollHeight, - }; - }); - - await stagehand.close(); - - const threeQuartersScroll = scrollInfo.scrollHeight * 0.75; - const threeQuartersReached = - Math.abs(scrollInfo.scrollTop - threeQuartersScroll) <= 200; - const evaluationResult = threeQuartersReached - ? { - _success: true, - logs: logger.getLogs(), - debugUrl, - sessionUrl, - } - : { - _success: false, - logs: logger.getLogs(), - debugUrl, - sessionUrl, - message: `Scroll position (${scrollInfo.scrollTop}px) is not three quarters down the page (${threeQuartersScroll}px).`, - }; - - return evaluationResult; -}; diff --git a/evals/tasks/simple_google_search.ts b/evals/tasks/simple_google_search.ts deleted file mode 100644 index e6a681200..000000000 --- a/evals/tasks/simple_google_search.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { EvalFunction } from "@/types/evals"; - -export const simple_google_search: EvalFunction = async ({ - debugUrl, - sessionUrl, - stagehand, - logger, -}) => { - await stagehand.page.goto("https://www.google.com"); - - await stagehand.page.act({ - action: 'type "OpenAI" into the search bar', - }); - - await stagehand.page.act("click the search button"); - - const expectedUrl = "https://www.google.com/search?q=OpenAI"; - const currentUrl = stagehand.page.url(); - - await stagehand.close(); - - return { - _success: currentUrl.startsWith(expectedUrl), - currentUrl, - debugUrl, - sessionUrl, - logs: logger.getLogs(), - }; -}; diff --git a/evals/tasks/stock_x.ts b/evals/tasks/stock_x.ts deleted file mode 100644 index b1459b0a6..000000000 --- a/evals/tasks/stock_x.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { EvalFunction } from "@/types/evals"; - -export const stock_x: EvalFunction = async ({ - debugUrl, - sessionUrl, - stagehand, - logger, -}) => { - await stagehand.page.goto( - "https://stockx.com/air-jordan-3-retro-black-cement-2024", - ); - - await stagehand.page.waitForTimeout(3000); - - await stagehand.page.act({ - action: "click on Jordan 3 Retro Crimson in the related products", - }); - - await stagehand.page.waitForTimeout(2000); - const currentUrl = stagehand.page.url(); - const expectedUrlPrefix = "https://stockx.com/jordan-3-retro-crimson"; - - await stagehand.close(); - - return { - _success: currentUrl.startsWith(expectedUrlPrefix), - currentUrl, - debugUrl, - sessionUrl, - logs: logger.getLogs(), - }; -}; diff --git a/evals/tasks/ted_talk.ts b/evals/tasks/ted_talk.ts deleted file mode 100644 index d288df8f6..000000000 --- a/evals/tasks/ted_talk.ts +++ /dev/null @@ -1,114 +0,0 @@ -import { EvalFunction } from "@/types/evals"; -import { normalizeString } from "@/evals/utils"; -import { z } from "zod"; - -export const ted_talk: EvalFunction = async ({ - debugUrl, - sessionUrl, - stagehand, - logger, - useTextExtract, -}) => { - await stagehand.page.goto( - "https://www.ted.com/talks/sir_ken_robinson_do_schools_kill_creativity", - { - waitUntil: "domcontentloaded", - }, - ); - await stagehand.page.act({ - action: - "Click the link that takes you to the page about the 'Culture' topic", - }); - - const playlists = await stagehand.page.extract({ - instruction: - "Extract the video playlist titles and the number of talks in each playlist. This info is in the Video Playlists about Culture section of the webpage.", - schema: z.object({ - playlists: z - .array( - z.object({ - title: z.string().describe("Title of the playlist"), - num_talks: z.number().describe("Number of talks in the playlist"), - }), - ) - .describe("List of culture video playlists"), - }), - useTextExtract, - }); - - await stagehand.close(); - - const expectedPlaylists = [ - { - title: "Talks that celebrate the boundless creativity of an open mind", - num_talks: 6, - }, - { - title: "Little-known big history", - num_talks: 15, - }, - { - title: "Extraordinary, larger-than-life art", - num_talks: 10, - }, - { - title: "How perfectionism fails us", - num_talks: 4, - }, - ]; - - if (!playlists.playlists || playlists.playlists.length === 0) { - logger.error({ - message: "Failed to extract playlists on culture", - level: 0, - }); - - return { - _success: false, - logs: logger.getLogs(), - debugUrl, - sessionUrl, - }; - } - - const missingPlaylists = expectedPlaylists.filter((expected) => - playlists.playlists.every( - (extracted) => - normalizeString(extracted.title) !== normalizeString(expected.title) || - extracted.num_talks !== expected.num_talks, - ), - ); - - if (missingPlaylists.length > 0) { - logger.error({ - message: "Extracted playlists do not match expected playlists", - level: 0, - auxiliary: { - missing: { - value: JSON.stringify(missingPlaylists), - type: "object", - }, - extracted: { - value: JSON.stringify(playlists.playlists), - type: "object", - }, - }, - }); - - return { - _success: false, - error: "Extracted playlists do not match expected playlists", - logs: logger.getLogs(), - debugUrl, - sessionUrl, - }; - } - - return { - _success: true, - playlists: playlists.playlists, - logs: logger.getLogs(), - debugUrl, - sessionUrl, - }; -}; diff --git a/evals/tasks/vanta_h.ts b/evals/tasks/vanta_h.ts deleted file mode 100644 index dba21d303..000000000 --- a/evals/tasks/vanta_h.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { EvalFunction } from "@/types/evals"; - -export const vanta_h: EvalFunction = async ({ - debugUrl, - sessionUrl, - stagehand, - logger, -}) => { - await stagehand.page.goto("https://www.vanta.com/"); - - const observations = await stagehand.page.observe( - "click the buy now button if it is available", - ); - - await stagehand.close(); - - // we should have no saved observation since the element shouldn't exist - return { - _success: observations.length === 0, - observations, - debugUrl, - sessionUrl, - logs: logger.getLogs(), - }; -}; diff --git a/evals/tasks/vantechjournal.ts b/evals/tasks/vantechjournal.ts deleted file mode 100644 index c176142b0..000000000 --- a/evals/tasks/vantechjournal.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { EvalFunction } from "@/types/evals"; - -export const vantechjournal: EvalFunction = async ({ - debugUrl, - sessionUrl, - stagehand, - logger, -}) => { - await stagehand.page.goto("https://vantechjournal.com/"); - - await stagehand.page.act({ - action: "click on page 8. do not click the next button", - }); - - const expectedUrl = "https://vantechjournal.com/archive?page=8"; - const currentUrl = stagehand.page.url(); - - await stagehand.close(); - - return { - _success: currentUrl === expectedUrl, - currentUrl, - expectedUrl, - debugUrl, - sessionUrl, - logs: logger.getLogs(), - }; -}; diff --git a/evals/tasks/wichita.ts b/evals/tasks/wichita.ts deleted file mode 100644 index fe645b480..000000000 --- a/evals/tasks/wichita.ts +++ /dev/null @@ -1,68 +0,0 @@ -import { EvalFunction } from "@/types/evals"; -import { z } from "zod"; - -export const wichita: EvalFunction = async ({ - debugUrl, - sessionUrl, - stagehand, - logger, - useTextExtract, -}) => { - await stagehand.page.goto("https://www.wichitafallstx.gov/Bids.aspx"); - - await stagehand.page.act({ - action: 'Click on "Show Closed/Awarded/Cancelled bids"', - }); - - const result = await stagehand.page.extract({ - instruction: "Extract the total number of bids that the search produced.", - schema: z.object({ - total_results: z.string(), - }), - useTextExtract, - }); - - await stagehand.close(); - - const { total_results } = result; - - const expectedNumber = 405; - const extractedNumber = parseInt(total_results.replace(/[^\d]/g, ""), 10); - - const isWithinRange = - extractedNumber >= expectedNumber - 10 && - extractedNumber <= expectedNumber + 10; - - if (!isWithinRange) { - logger.error({ - message: "Total number of results is not within the expected range", - level: 0, - auxiliary: { - expected: { - value: `${expectedNumber} ± 10`, - type: "string", - }, - actual: { - value: extractedNumber.toString(), - type: "integer", - }, - }, - }); - return { - _success: false, - error: "Total number of results is not within the expected range", - extractedNumber, - debugUrl, - sessionUrl, - logs: logger.getLogs(), - }; - } - - return { - _success: true, - extractedNumber, - debugUrl, - sessionUrl, - logs: logger.getLogs(), - }; -}; diff --git a/evals/tasks/wikipedia.ts b/evals/tasks/wikipedia.ts deleted file mode 100644 index 5dad05d2f..000000000 --- a/evals/tasks/wikipedia.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { EvalFunction } from "@/types/evals"; - -export const wikipedia: EvalFunction = async ({ - debugUrl, - sessionUrl, - stagehand, - logger, -}) => { - await stagehand.page.goto(`https://en.wikipedia.org/wiki/Baseball`); - await stagehand.page.act({ - action: 'click the "hit and run" link in this article', - timeoutMs: 360_000, - }); - - const url = "https://en.wikipedia.org/wiki/Hit_and_run_(baseball)"; - const currentUrl = stagehand.page.url(); - - await stagehand.close(); - - return { - _success: currentUrl === url, - expected: url, - actual: currentUrl, - debugUrl, - sessionUrl, - logs: logger.getLogs(), - }; -}; diff --git a/evals/utils.ts b/evals/utils.ts deleted file mode 100644 index a573d1729..000000000 --- a/evals/utils.ts +++ /dev/null @@ -1,121 +0,0 @@ -/** - * This file provides utility functions and classes to assist with evaluation tasks. - * - * Key functionalities: - * - String normalization and fuzzy comparison utility functions to compare output strings - * against expected results in a flexible and robust way. - * - Generation of unique experiment names based on the current timestamp, environment, - * and eval name or category. - */ - -import { LogLine } from "@/dist"; -import stringComparison from "string-comparison"; -const { jaroWinkler } = stringComparison; - -/** - * normalizeString: - * Prepares a string for comparison by: - * - Converting to lowercase - * - Collapsing multiple spaces to a single space - * - Removing punctuation and special characters that are not alphabetic or numeric - * - Normalizing spacing around commas - * - Trimming leading and trailing whitespace - * - * This helps create a stable string representation to compare against expected outputs, - * even if the actual output contains minor formatting differences. - */ -export function normalizeString(str: string): string { - return str - .toLowerCase() - .replace(/\s+/g, " ") - .replace(/[;/#!$%^&*:{}=\-_`~()]/g, "") - .replace(/\s*,\s*/g, ", ") - .trim(); -} - -/** - * compareStrings: - * Compares two strings (actual vs. expected) using a similarity metric (Jaro-Winkler). - * - * Arguments: - * - actual: The actual output string to be checked. - * - expected: The expected string we want to match against. - * - similarityThreshold: A number between 0 and 1. Default is 0.85. - * If the computed similarity is greater than or equal to this threshold, - * we consider the strings sufficiently similar. - * - * Returns: - * - similarity: A number indicating how similar the two strings are. - * - meetsThreshold: A boolean indicating if the similarity meets or exceeds the threshold. - * - * This function is useful for tasks where exact string matching is too strict, - * allowing for fuzzy matching that tolerates minor differences in formatting or spelling. - */ -export function compareStrings( - actual: string, - expected: string, - similarityThreshold: number = 0.85, -): { similarity: number; meetsThreshold: boolean } { - const similarity = jaroWinkler.similarity( - normalizeString(actual), - normalizeString(expected), - ); - return { - similarity, - meetsThreshold: similarity >= similarityThreshold, - }; -} - -/** - * generateTimestamp: - * Generates a timestamp string formatted as "YYYYMMDDHHMMSS". - * Used to create unique experiment names, ensuring that results can be - * distinguished by the time they were generated. - */ -export function generateTimestamp(): string { - const now = new Date(); - return now - .toISOString() - .replace(/[-:TZ]/g, "") - .slice(0, 14); -} - -/** - * generateExperimentName: - * Creates a unique name for the experiment based on optional evalName or category, - * the environment (e.g., dev or CI), and the current timestamp. - * This is used to label the output files and directories. - */ -export function generateExperimentName({ - evalName, - category, - environment, -}: { - evalName?: string; - category?: string; - environment: string; -}): string { - const timestamp = generateTimestamp(); - if (evalName) { - return `${evalName}_${environment.toLowerCase()}_${timestamp}`; - } - if (category) { - return `${category}_${environment.toLowerCase()}_${timestamp}`; - } - return `all_${environment.toLowerCase()}_${timestamp}`; -} - -export function logLineToString(logLine: LogLine): string { - try { - const timestamp = logLine.timestamp || new Date().toISOString(); - if (logLine.auxiliary?.error) { - return `${timestamp}::[stagehand:${logLine.category}] ${logLine.message}\n ${logLine.auxiliary.error.value}\n ${logLine.auxiliary.trace.value}`; - } - return `${timestamp}::[stagehand:${logLine.category}] ${logLine.message} ${ - logLine.auxiliary ? JSON.stringify(logLine.auxiliary) : "" - }`; - } catch (error) { - console.error(`Error logging line:`, error); - return "error logging line"; - } -} diff --git a/examples/actionable_observe_example.ts b/examples/actionable_observe_example.ts deleted file mode 100644 index 5fa55d80c..000000000 --- a/examples/actionable_observe_example.ts +++ /dev/null @@ -1,71 +0,0 @@ -/** - * This file is meant to be used as a scratchpad for trying out actionable observe. - * To create a Stagehand project with best practices and configuration, run: - * - * npx create-browser-app@latest my-browser-app - */ - -import { Stagehand } from "@/dist"; -import stagehandConfig from "@/stagehand.config"; - -async function example() { - const stagehand = new Stagehand(stagehandConfig); - await stagehand.init(); - await stagehand.page.goto("https://www.apartments.com/san-francisco-ca/"); - - await new Promise((resolve) => setTimeout(resolve, 3000)); - const observations1 = await stagehand.page.observe({ - instruction: "find the 'all filters' button", - }); - await stagehand.page.act(observations1[0]); - - await new Promise((resolve) => setTimeout(resolve, 3000)); - const observations2 = await stagehand.page.observe({ - instruction: "find the '1+' button in the 'beds' section", - }); - await stagehand.page.act(observations2[0]); - - await new Promise((resolve) => setTimeout(resolve, 3000)); - const observations3 = await stagehand.page.observe({ - instruction: "find the 'apartments' button in the 'home type' section", - }); - await stagehand.page.act(observations3[0]); - - await new Promise((resolve) => setTimeout(resolve, 3000)); - const observations4 = await stagehand.page.observe({ - instruction: "find the pet policy dropdown to click on.", - }); - await stagehand.page.act(observations4[0]); - - await new Promise((resolve) => setTimeout(resolve, 3000)); - const observations5 = await stagehand.page.observe({ - instruction: "find the 'Dog Friendly' option to click on", - }); - await stagehand.page.act(observations5[0]); - - await new Promise((resolve) => setTimeout(resolve, 3000)); - const observations6 = await stagehand.page.observe({ - instruction: "find the 'see results' section", - }); - await stagehand.page.act(observations6[0]); - - const currentUrl = await stagehand.page.url(); - await stagehand.close(); - if ( - currentUrl.includes( - "https://www.apartments.com/apartments/san-francisco-ca/min-1-bedrooms-pet-friendly-dog/", - ) - ) { - console.log("✅ Success! we made it to the correct page"); - } else { - console.log( - "❌ Whoops, looks like we didn't make it to the correct page. " + - "\nThanks for testing out this new Stagehand feature!" + - "\nReach us on Slack if you have any feedback/questions/suggestions!", - ); - } -} - -(async () => { - await example(); -})(); diff --git a/examples/ai_sdk_example.ts b/examples/ai_sdk_example.ts deleted file mode 100644 index b650de5c1..000000000 --- a/examples/ai_sdk_example.ts +++ /dev/null @@ -1,34 +0,0 @@ -import { openai } from "@ai-sdk/openai"; -import { Stagehand } from "@/dist"; -import { AISdkClient } from "./external_clients/aisdk"; -import StagehandConfig from "@/stagehand.config"; -import { z } from "zod"; - -async function example() { - const stagehand = new Stagehand({ - ...StagehandConfig, - llmClient: new AISdkClient({ - model: openai("gpt-4o"), - }), - }); - - await stagehand.init(); - await stagehand.page.goto("https://news.ycombinator.com"); - - const { story } = await stagehand.page.extract({ - instruction: "extract the title of the top story on the page", - schema: z.object({ - story: z.string().describe("the top story on the page"), - }), - }); - - console.log("The top story is:", story); - - await stagehand.page.act("click the first story"); - - await stagehand.close(); -} - -(async () => { - await example(); -})(); diff --git a/examples/cua-example.ts b/examples/cua-example.ts deleted file mode 100644 index e48eaf25b..000000000 --- a/examples/cua-example.ts +++ /dev/null @@ -1,91 +0,0 @@ -import { Stagehand } from "@/dist"; -import dotenv from "dotenv"; -import StagehandConfig from "@/stagehand.config"; -import chalk from "chalk"; - -// Load environment variables -dotenv.config(); - -async function main() { - console.log( - `\n${chalk.bold("Stagehand 🤘 Computer Use Agent (CUA) Demo")}\n`, - ); - - // Initialize Stagehand - console.log(`${chalk.cyan("→")} Initializing Stagehand...`); - const stagehand = new Stagehand({ - ...StagehandConfig, - }); - - await stagehand.init(); - console.log(`${chalk.green("✓")} Stagehand initialized`); - - try { - const page = stagehand.page; - - console.log(`\n${chalk.magenta.bold("⚡ First Agent Execution")}`); - - const agent = stagehand.agent({ - provider: "openai", - model: "computer-use-preview", - instructions: `You are a helpful assistant that can use a web browser. - You are currently on the following page: ${page.url()}. - Do not ask follow up questions, the user will trust your judgement.`, - options: { - apiKey: process.env.OPENAI_API_KEY, - }, - }); - - console.log(`${chalk.yellow("→")} Navigating to Google...`); - await stagehand.page.goto("https://www.google.com"); - console.log(`${chalk.green("✓")} Loaded: ${chalk.dim(page.url())}`); - - const firstInstruction = - "Search for openai news on google and extract the name of the first 3 results"; - console.log( - `${chalk.cyan("↳")} Instruction: ${chalk.white(firstInstruction)}`, - ); - - const result1 = await agent.execute(firstInstruction); - - console.log(`${chalk.green("✓")} Execution complete`); - console.log(`${chalk.yellow("⤷")} Result:`); - console.log(chalk.white(JSON.stringify(result1, null, 2))); - - console.log(`\n${chalk.magenta.bold("⚡ Second Agent Execution")}`); - - console.log( - `\n${chalk.yellow("→")} Navigating to Browserbase careers page...`, - ); - await page.goto("https://www.browserbase.com/careers"); - console.log(`${chalk.green("✓")} Loaded: ${chalk.dim(page.url())}`); - - const instruction = - "Apply for the first engineer position with mock data. Don't submit the form."; - console.log(`${chalk.cyan("↳")} Instruction: ${chalk.white(instruction)}`); - - const result = await agent.execute({ - instruction, - maxSteps: 20, - }); - - console.log(`${chalk.green("✓")} Execution complete`); - console.log(`${chalk.yellow("⤷")} Result:`); - console.log(chalk.white(JSON.stringify(result, null, 2))); - } catch (error) { - console.log(`${chalk.red("✗")} Error: ${error}`); - if (error instanceof Error && error.stack) { - console.log(chalk.dim(error.stack.split("\n").slice(1).join("\n"))); - } - } finally { - // Close the browser - console.log(`\n${chalk.yellow("→")} Closing browser...`); - await stagehand.close(); - console.log(`${chalk.green("✓")} Browser closed\n`); - } -} - -main().catch((error) => { - console.log(`${chalk.red("✗")} Unhandled error in main function`); - console.log(chalk.red(error)); -}); diff --git a/examples/debugUrl.ts b/examples/debugUrl.ts deleted file mode 100644 index e54bb1e2a..000000000 --- a/examples/debugUrl.ts +++ /dev/null @@ -1,24 +0,0 @@ -import { Stagehand } from "@/dist"; - -async function debug(url: string) { - const stagehand = new Stagehand({ - env: "LOCAL", - verbose: 2, - localBrowserLaunchOptions: { - headless: true, - }, - }); - await stagehand.init(); - await stagehand.page.goto(url); -} - -(async () => { - const url = process.argv.find((arg) => arg.startsWith("--url=")); - if (!url) { - console.error("No URL flag provided. Usage: --url=https://example.com"); - process.exit(1); - } - const targetUrl = url.split("=")[1]; - console.log(`Navigating to: ${targetUrl}`); - await debug(targetUrl); -})(); diff --git a/examples/example.ts b/examples/example.ts deleted file mode 100644 index 372d72e5c..000000000 --- a/examples/example.ts +++ /dev/null @@ -1,25 +0,0 @@ -/** - * This file is meant to be used as a scratchpad for developing new evals. - * To create a Stagehand project with best practices and configuration, run: - * - * npx create-browser-app@latest my-browser-app - */ - -import { Stagehand } from "@/dist"; -import StagehandConfig from "@/stagehand.config"; - -async function example() { - const stagehand = new Stagehand({ - ...StagehandConfig, - }); - await stagehand.init(); - await stagehand.page.goto("https://docs.stagehand.dev"); - /** - * Add your code here! - */ - await stagehand.close(); -} - -(async () => { - await example(); -})(); diff --git a/examples/external_client.ts b/examples/external_client.ts deleted file mode 100644 index 63f92256a..000000000 --- a/examples/external_client.ts +++ /dev/null @@ -1,42 +0,0 @@ -import { Stagehand } from "@/dist"; -import { z } from "zod"; -import { CustomOpenAIClient } from "./external_clients/customOpenAI"; -import StagehandConfig from "@/stagehand.config"; -import OpenAI from "openai"; - -async function example() { - const stagehand = new Stagehand({ - ...StagehandConfig, - llmClient: new CustomOpenAIClient({ - modelName: "gpt-4o-mini", - client: new OpenAI({ - apiKey: process.env.OPENAI_API_KEY, - }), - }), - }); - - await stagehand.init(); - await stagehand.page.goto("https://news.ycombinator.com"); - await stagehand.page.act("click on the 'new' link"); - - const headlines = await stagehand.page.extract({ - instruction: "Extract the top 3 stories from the Hacker News homepage.", - schema: z.object({ - stories: z.array( - z.object({ - title: z.string(), - url: z.string(), - points: z.number(), - }), - ), - }), - }); - - console.log(headlines); - - await stagehand.close(); -} - -(async () => { - await example(); -})(); diff --git a/examples/external_clients/aisdk.ts b/examples/external_clients/aisdk.ts deleted file mode 100644 index 1d72d984f..000000000 --- a/examples/external_clients/aisdk.ts +++ /dev/null @@ -1,122 +0,0 @@ -import { - CoreAssistantMessage, - CoreMessage, - CoreSystemMessage, - CoreTool, - CoreUserMessage, - generateObject, - generateText, - ImagePart, - LanguageModel, - TextPart, -} from "ai"; -import { CreateChatCompletionOptions, LLMClient, AvailableModel } from "@/dist"; -import { ChatCompletion } from "openai/resources"; - -export class AISdkClient extends LLMClient { - public type = "aisdk" as const; - private model: LanguageModel; - - constructor({ model }: { model: LanguageModel }) { - super(model.modelId as AvailableModel); - this.model = model; - } - - async createChatCompletion({ - options, - }: CreateChatCompletionOptions): Promise { - const formattedMessages: CoreMessage[] = options.messages.map((message) => { - if (Array.isArray(message.content)) { - if (message.role === "system") { - const systemMessage: CoreSystemMessage = { - role: "system", - content: message.content - .map((c) => ("text" in c ? c.text : "")) - .join("\n"), - }; - return systemMessage; - } - - const contentParts = message.content.map((content) => { - if ("image_url" in content) { - const imageContent: ImagePart = { - type: "image", - image: content.image_url.url, - }; - return imageContent; - } else { - const textContent: TextPart = { - type: "text", - text: content.text, - }; - return textContent; - } - }); - - if (message.role === "user") { - const userMessage: CoreUserMessage = { - role: "user", - content: contentParts, - }; - return userMessage; - } else { - const textOnlyParts = contentParts.map((part) => ({ - type: "text" as const, - text: part.type === "image" ? "[Image]" : part.text, - })); - const assistantMessage: CoreAssistantMessage = { - role: "assistant", - content: textOnlyParts, - }; - return assistantMessage; - } - } - - return { - role: message.role, - content: message.content, - }; - }); - - if (options.response_model) { - const response = await generateObject({ - model: this.model, - messages: formattedMessages, - schema: options.response_model.schema, - }); - - return { - data: response.object, - usage: { - prompt_tokens: response.usage.promptTokens ?? 0, - completion_tokens: response.usage.completionTokens ?? 0, - total_tokens: response.usage.totalTokens ?? 0, - }, - } as T; - } - - const tools: Record = {}; - - for (const rawTool of options.tools) { - tools[rawTool.name] = { - description: rawTool.description, - parameters: rawTool.parameters, - }; - } - - const response = await generateText({ - model: this.model, - messages: formattedMessages, - tools, - }); - - return { - data: response.text, - usage: { - prompt_tokens: response.usage.promptTokens ?? 0, - completion_tokens: response.usage.completionTokens ?? 0, - total_tokens: response.usage.totalTokens ?? 0, - }, - } as T; - } -} diff --git a/examples/external_clients/customOpenAI.ts b/examples/external_clients/customOpenAI.ts deleted file mode 100644 index 6a6d70b3f..000000000 --- a/examples/external_clients/customOpenAI.ts +++ /dev/null @@ -1,240 +0,0 @@ -/** - * Welcome to the Stagehand custom OpenAI client! - * - * This is a client for models that are compatible with the OpenAI API, like Ollama, Gemini, etc. - * You can just pass in an OpenAI instance to the client and it will work. - */ - -import { AvailableModel, CreateChatCompletionOptions, LLMClient } from "@/dist"; -import OpenAI from "openai"; -import { zodResponseFormat } from "openai/helpers/zod"; -import type { - ChatCompletion, - ChatCompletionAssistantMessageParam, - ChatCompletionContentPartImage, - ChatCompletionContentPartText, - ChatCompletionCreateParamsNonStreaming, - ChatCompletionMessageParam, - ChatCompletionSystemMessageParam, - ChatCompletionUserMessageParam, -} from "openai/resources/chat/completions"; -import { z } from "zod"; -import { CreateChatCompletionResponseError } from "@/types/stagehandErrors"; - -function validateZodSchema(schema: z.ZodTypeAny, data: unknown) { - try { - schema.parse(data); - return true; - } catch { - return false; - } -} - -export class CustomOpenAIClient extends LLMClient { - public type = "openai" as const; - private client: OpenAI; - - constructor({ modelName, client }: { modelName: string; client: OpenAI }) { - super(modelName as AvailableModel); - this.client = client; - this.modelName = modelName as AvailableModel; - } - - async createChatCompletion({ - options, - retries = 3, - logger, - }: CreateChatCompletionOptions): Promise { - const { image, requestId, ...optionsWithoutImageAndRequestId } = options; - - // TODO: Implement vision support - if (image) { - console.warn( - "Image provided. Vision is not currently supported for openai", - ); - } - - logger({ - category: "openai", - message: "creating chat completion", - level: 1, - auxiliary: { - options: { - value: JSON.stringify({ - ...optionsWithoutImageAndRequestId, - requestId, - }), - type: "object", - }, - modelName: { - value: this.modelName, - type: "string", - }, - }, - }); - - if (options.image) { - console.warn( - "Image provided. Vision is not currently supported for openai", - ); - } - - let responseFormat = undefined; - if (options.response_model) { - responseFormat = zodResponseFormat( - options.response_model.schema, - options.response_model.name, - ); - } - - /* eslint-disable */ - // Remove unsupported options - const { response_model, ...openaiOptions } = { - ...optionsWithoutImageAndRequestId, - model: this.modelName, - }; - - logger({ - category: "openai", - message: "creating chat completion", - level: 1, - auxiliary: { - openaiOptions: { - value: JSON.stringify(openaiOptions), - type: "object", - }, - }, - }); - - const formattedMessages: ChatCompletionMessageParam[] = - options.messages.map((message) => { - if (Array.isArray(message.content)) { - const contentParts = message.content.map((content) => { - if ("image_url" in content) { - const imageContent: ChatCompletionContentPartImage = { - image_url: { - url: content.image_url.url, - }, - type: "image_url", - }; - return imageContent; - } else { - const textContent: ChatCompletionContentPartText = { - text: content.text, - type: "text", - }; - return textContent; - } - }); - - if (message.role === "system") { - const formattedMessage: ChatCompletionSystemMessageParam = { - ...message, - role: "system", - content: contentParts.filter( - (content): content is ChatCompletionContentPartText => - content.type === "text", - ), - }; - return formattedMessage; - } else if (message.role === "user") { - const formattedMessage: ChatCompletionUserMessageParam = { - ...message, - role: "user", - content: contentParts, - }; - return formattedMessage; - } else { - const formattedMessage: ChatCompletionAssistantMessageParam = { - ...message, - role: "assistant", - content: contentParts.filter( - (content): content is ChatCompletionContentPartText => - content.type === "text", - ), - }; - return formattedMessage; - } - } - - const formattedMessage: ChatCompletionUserMessageParam = { - role: "user", - content: message.content, - }; - - return formattedMessage; - }); - - const body: ChatCompletionCreateParamsNonStreaming = { - ...openaiOptions, - model: this.modelName, - messages: formattedMessages, - response_format: responseFormat, - stream: false, - tools: options.tools?.map((tool) => ({ - function: { - name: tool.name, - description: tool.description, - parameters: tool.parameters, - }, - type: "function", - })), - }; - - const response = await this.client.chat.completions.create(body); - - logger({ - category: "openai", - message: "response", - level: 1, - auxiliary: { - response: { - value: JSON.stringify(response), - type: "object", - }, - requestId: { - value: requestId, - type: "string", - }, - }, - }); - - if (options.response_model) { - const extractedData = response.choices[0].message.content; - if (!extractedData) { - throw new CreateChatCompletionResponseError("No content in response"); - } - const parsedData = JSON.parse(extractedData); - - if (!validateZodSchema(options.response_model.schema, parsedData)) { - if (retries > 0) { - return this.createChatCompletion({ - options, - logger, - retries: retries - 1, - }); - } - - throw new CreateChatCompletionResponseError("Invalid response schema"); - } - - return { - data: parsedData, - usage: { - prompt_tokens: response.usage?.prompt_tokens ?? 0, - completion_tokens: response.usage?.completion_tokens ?? 0, - total_tokens: response.usage?.total_tokens ?? 0, - }, - } as T; - } - - return { - data: response.choices[0].message.content, - usage: { - prompt_tokens: response.usage?.prompt_tokens ?? 0, - completion_tokens: response.usage?.completion_tokens ?? 0, - total_tokens: response.usage?.total_tokens ?? 0, - }, - } as T; - } -} diff --git a/examples/external_clients/langchain.ts b/examples/external_clients/langchain.ts deleted file mode 100644 index 1d071a63b..000000000 --- a/examples/external_clients/langchain.ts +++ /dev/null @@ -1,87 +0,0 @@ -import { BaseChatModel } from "@langchain/core/language_models/chat_models"; -import { CreateChatCompletionOptions, LLMClient, AvailableModel } from "@/dist"; -import { zodToJsonSchema } from "zod-to-json-schema"; -import { - AIMessage, - BaseMessageLike, - HumanMessage, - SystemMessage, -} from "@langchain/core/messages"; -import { ChatCompletion } from "openai/resources"; - -export class LangchainClient extends LLMClient { - public type = "langchainClient" as const; - private model: BaseChatModel; - - constructor(model: BaseChatModel) { - super(model.name as AvailableModel); - this.model = model; - } - - async createChatCompletion({ - options, - }: CreateChatCompletionOptions): Promise { - const formattedMessages: BaseMessageLike[] = options.messages.map( - (message) => { - if (Array.isArray(message.content)) { - if (message.role === "system") { - return new SystemMessage( - message.content - .map((c) => ("text" in c ? c.text : "")) - .join("\n"), - ); - } - - const content = message.content.map((content) => - "image_url" in content - ? { type: "image", image: content.image_url.url } - : { type: "text", text: content.text }, - ); - - if (message.role === "user") return new HumanMessage({ content }); - - const textOnlyParts = content.map((part) => ({ - type: "text" as const, - text: part.type === "image" ? "[Image]" : part.text, - })); - - return new AIMessage({ content: textOnlyParts }); - } - - return { - role: message.role, - content: message.content, - }; - }, - ); - - if (options.response_model) { - const responseSchema = zodToJsonSchema(options.response_model.schema, { - $refStrategy: "none", - }); - const structuredModel = this.model.withStructuredOutput(responseSchema); - const response = await structuredModel.invoke(formattedMessages); - - return { - data: response, - usage: { - prompt_tokens: 0, // Langchain doesn't provide token counts by default - completion_tokens: 0, - total_tokens: 0, - }, - } as T; - } - - const modelWithTools = this.model.bindTools(options.tools); - const response = await modelWithTools.invoke(formattedMessages); - - return { - data: response, - usage: { - prompt_tokens: 0, // Langchain doesn't provide token counts by default - completion_tokens: 0, - total_tokens: 0, - }, - } as T; - } -} diff --git a/examples/form_filling_sensible.ts b/examples/form_filling_sensible.ts deleted file mode 100644 index e2de10dd7..000000000 --- a/examples/form_filling_sensible.ts +++ /dev/null @@ -1,91 +0,0 @@ -import { Stagehand } from "@/dist"; -import StagehandConfig from "@/stagehand.config"; -import chalk from "chalk"; - -async function formFillingSensible() { - const stagehand = new Stagehand({ - ...StagehandConfig, - // Uncomment the following lines to run locally or use a different model - env: "LOCAL", - modelName: "gpt-4o-mini", - }); - await stagehand.init(); - - // Block manifest worker to prevent PWA installation popup. - // This is necessary because the website prompts the user to install the PWA and prevents form filling. - await stagehand.page.route("**/manifest.json", (route) => route.abort()); - - // Go to the website and wait for it to load - await stagehand.page.goto("https://file.1040.com/estimate/", { - waitUntil: "networkidle", - timeout: 30000, - }); - - // Observe the form fields with suggested actions - const observed = await stagehand.page.observe({ - instruction: - "fill all the form fields in the page with mock data. In the description inlcude the field name", - returnAction: true, - }); - - // Uncomment the following snippet to see the stagehand candidate suggestions (initial) - console.log( - `${chalk.green("Observe:")} Form fields found:\n${observed - .map((r) => `${chalk.yellow(r.description)} -> ${chalk.gray(r.selector)}`) - .join("\n")}`, - ); - - // Create a mapping of 1+ keywords in the form fields to standardize field names - const mapping = (description: string): string | null => { - const keywords: { [key: string]: string[] } = { - age: ["old"], - dependentsUnder17: ["under age 17", "child", "minor"], - dependents17to23: ["17-23", "school", "student"], - wages: ["wages", "W-2 Box 1"], - federalTax: ["federal tax", "Box 2"], - stateTax: ["state tax", "Box 17"], - }; - - for (const [key, terms] of Object.entries(keywords)) { - if (terms.some((term) => description.toLowerCase().includes(term))) { - return key; - } - } - return null; - }; - - // Fill the form fields with sensible data. This data will only be used in your session and not be shared with LLM providers/external APIs. - const userInputs: { [key: string]: string } = { - age: "26", - dependentsUnder17: "1", - wages: "54321", - federalTax: "8345", - stateTax: "2222", - }; - - const updatedFields = observed.map((candidate) => { - const key = mapping(candidate.description); - if (key && userInputs[key]) { - candidate.arguments = [userInputs[key]]; - } - return candidate; - }); - // List of sensible-data candidates - console.log( - `\n${chalk.green("Sensible Data form inputs:")} Form fields to be filled:\n${updatedFields - .map( - (r) => - `${chalk.yellow(r.description)} -> ${chalk.blue(r.arguments?.[0] || "no value")}`, - ) - .join("\n")}`, - ); - - // Fill all the form fields with the sensible candidates - for (const candidate of updatedFields) { - await stagehand.page.act(candidate); - } -} - -(async () => { - await formFillingSensible(); -})(); diff --git a/examples/form_filling_sensible_cerebras.ts b/examples/form_filling_sensible_cerebras.ts deleted file mode 100644 index 862a3e471..000000000 --- a/examples/form_filling_sensible_cerebras.ts +++ /dev/null @@ -1,97 +0,0 @@ -import { Stagehand } from "@/dist"; -import StagehandConfig from "@/stagehand.config"; -import { CerebrasClient } from "../lib/llm/CerebrasClient"; -import chalk from "chalk"; - -async function formFillingSensible() { - const stagehand = new Stagehand({ - ...StagehandConfig, - env: "LOCAL", - llmClient: new CerebrasClient({ - modelName: "cerebras-llama-3.3-70b", - clientOptions: { - apiKey: process.env.CEREBRAS_API_KEY, - }, - logger: console.log, - }), - }); - await stagehand.init(); - - // Block manifest worker to prevent PWA installation popup. - // This is necessary because the website prompts the user to install the PWA and prevents form filling. - await stagehand.page.route("**/manifest.json", (route) => route.abort()); - - // Go to the website and wait for it to load - await stagehand.page.goto("https://file.1040.com/estimate/", { - waitUntil: "networkidle", - timeout: 30000, - }); - - // Observe the form fields with suggested actions - const observed = await stagehand.page.observe({ - instruction: - "fill all the form fields in the page with mock data. In the description inlcude the field name", - returnAction: true, - }); - - // Uncomment the following snippet to see the stagehand candidate suggestions (initial) - console.log( - `${chalk.green("Observe:")} Form fields found:\n${observed - .map((r) => `${chalk.yellow(r.description)} -> ${chalk.gray(r.selector)}`) - .join("\n")}`, - ); - - // Create a mapping of 1+ keywords in the form fields to standardize field names - const mapping = (description: string): string | null => { - const keywords: { [key: string]: string[] } = { - age: ["old"], - dependentsUnder17: ["under age 17", "child", "minor"], - dependents17to23: ["17-23", "school", "student"], - wages: ["wages", "W-2 Box 1"], - federalTax: ["federal tax", "Box 2"], - stateTax: ["state tax", "Box 17"], - }; - - for (const [key, terms] of Object.entries(keywords)) { - if (terms.some((term) => description.toLowerCase().includes(term))) { - return key; - } - } - return null; - }; - - // Fill the form fields with sensible data. This data will only be used in your session and not be shared with LLM providers/external APIs. - const userInputs: { [key: string]: string } = { - age: "26", - dependentsUnder17: "1", - wages: "54321", - federalTax: "8345", - stateTax: "2222", - }; - - const updatedFields = observed.map((candidate) => { - const key = mapping(candidate.description); - if (key && userInputs[key]) { - candidate.arguments = [userInputs[key]]; - } - return candidate; - }); - // List of sensible-data candidates - console.log( - `\n${chalk.green("Sensible Data form inputs:")} Form fields to be filled:\n${updatedFields - .map( - (r) => - `${chalk.yellow(r.description)} -> ${chalk.blue(r.arguments?.[0] || "no value")}`, - ) - .join("\n")}`, - ); - - // Fill all the form fields with the sensible candidates - for (const candidate of updatedFields) { - await stagehand.page.act(candidate); - } -} - -(async () => { - await formFillingSensible(); -})(); diff --git a/examples/form_filling_sensible_groq.ts b/examples/form_filling_sensible_groq.ts deleted file mode 100644 index cf885e215..000000000 --- a/examples/form_filling_sensible_groq.ts +++ /dev/null @@ -1,97 +0,0 @@ -import { Stagehand } from "@/dist"; -import { GroqClient } from "@/lib/llm/GroqClient"; -import StagehandConfig from "@/stagehand.config"; -import chalk from "chalk"; - -async function formFillingSensible() { - const stagehand = new Stagehand({ - ...StagehandConfig, - env: "LOCAL", - llmClient: new GroqClient({ - modelName: "groq-llama-3.3-70b-versatile", - clientOptions: { - apiKey: process.env.GROQ_API_KEY, - }, - logger: console.log, - }), - }); - await stagehand.init(); - - // Block manifest worker to prevent PWA installation popup. - // This is necessary because the website prompts the user to install the PWA and prevents form filling. - await stagehand.page.route("**/manifest.json", (route) => route.abort()); - - // Go to the website and wait for it to load - await stagehand.page.goto("https://file.1040.com/estimate/", { - waitUntil: "networkidle", - timeout: 30000, - }); - - // Observe the form fields with suggested actions - const observed = await stagehand.page.observe({ - instruction: - "fill all the form fields in the page with mock data. In the description inlcude the field name", - returnAction: true, - }); - - // Uncomment the following snippet to see the stagehand candidate suggestions (initial) - console.log( - `${chalk.green("Observe:")} Form fields found:\n${observed - .map((r) => `${chalk.yellow(r.description)} -> ${chalk.gray(r.selector)}`) - .join("\n")}`, - ); - - // Create a mapping of 1+ keywords in the form fields to standardize field names - const mapping = (description: string): string | null => { - const keywords: { [key: string]: string[] } = { - age: ["old"], - dependentsUnder17: ["under age 17", "child", "minor"], - dependents17to23: ["17-23", "school", "student"], - wages: ["wages", "W-2 Box 1"], - federalTax: ["federal tax", "Box 2"], - stateTax: ["state tax", "Box 17"], - }; - - for (const [key, terms] of Object.entries(keywords)) { - if (terms.some((term) => description.toLowerCase().includes(term))) { - return key; - } - } - return null; - }; - - // Fill the form fields with sensible data. This data will only be used in your session and not be shared with LLM providers/external APIs. - const userInputs: { [key: string]: string } = { - age: "26", - dependentsUnder17: "1", - wages: "54321", - federalTax: "8345", - stateTax: "2222", - }; - - const updatedFields = observed.map((candidate) => { - const key = mapping(candidate.description); - if (key && userInputs[key]) { - candidate.arguments = [userInputs[key]]; - } - return candidate; - }); - // List of sensible-data candidates - console.log( - `\n${chalk.green("Sensible Data form inputs:")} Form fields to be filled:\n${updatedFields - .map( - (r) => - `${chalk.yellow(r.description)} -> ${chalk.blue(r.arguments?.[0] || "no value")}`, - ) - .join("\n")}`, - ); - - // Fill all the form fields with the sensible candidates - for (const candidate of updatedFields) { - await stagehand.page.act(candidate); - } -} - -(async () => { - await formFillingSensible(); -})(); diff --git a/examples/google_enter.ts b/examples/google_enter.ts deleted file mode 100644 index affa1eb93..000000000 --- a/examples/google_enter.ts +++ /dev/null @@ -1,25 +0,0 @@ -/** - * This file is meant to be used as a scratchpad for developing new evals. - * To create a Stagehand project with best practices and configuration, run: - * - * npx create-browser-app@latest my-browser-app - */ - -import { Stagehand } from "@/dist"; -import StagehandConfig from "@/stagehand.config"; - -async function example() { - const stagehand = new Stagehand({ - ...StagehandConfig, - }); - await stagehand.init(); - const page = stagehand.page; - await page.goto("https://google.com"); - await page.act("type in 'Browserbase'"); - await page.act("press enter"); - await stagehand.close(); -} - -(async () => { - await example(); -})(); diff --git a/examples/instructions.ts b/examples/instructions.ts deleted file mode 100644 index 250199ba1..000000000 --- a/examples/instructions.ts +++ /dev/null @@ -1,32 +0,0 @@ -/** - * This example shows how to use custom instructions with Stagehand. - */ -import { Stagehand } from "@/dist"; -import StagehandConfig from "@/stagehand.config"; - -async function example() { - const stagehand = new Stagehand({ - ...StagehandConfig, - systemPrompt: - "if the users says `secret12345`, click on the 'getting started' tab. additionally, if the user says to type something, translate their input into french and type it.", - }); - await stagehand.init(); - - const page = stagehand.page; - - await page.goto("https://docs.browserbase.com/"); - - await page.act({ - action: "secret12345", - }); - - await page.act({ - action: "search for 'how to use browserbase'", - }); - - await stagehand.close(); -} - -(async () => { - await example(); -})(); diff --git a/examples/langchain.ts b/examples/langchain.ts deleted file mode 100644 index 98fe41d4c..000000000 --- a/examples/langchain.ts +++ /dev/null @@ -1,35 +0,0 @@ -import { z } from "zod"; -import { Stagehand } from "@/dist"; -import StagehandConfig from "@/stagehand.config"; -import { LangchainClient } from "./external_clients/langchain"; -import { ChatOpenAI } from "@langchain/openai"; - -async function example() { - const stagehand = new Stagehand({ - ...StagehandConfig, - llmClient: new LangchainClient( - new ChatOpenAI({ - model: "gpt-4o", - }), - ), - }); - - await stagehand.init(); - await stagehand.page.goto("https://news.ycombinator.com"); - - const { story } = await stagehand.page.extract({ - schema: z.object({ - story: z.string().describe("the top story on the page"), - }), - }); - - console.log("The top story is:", story); - - await stagehand.page.act("click the first story"); - - await stagehand.close(); -} - -(async () => { - await example(); -})(); diff --git a/examples/operator-example.ts b/examples/operator-example.ts deleted file mode 100644 index 5978e8d72..000000000 --- a/examples/operator-example.ts +++ /dev/null @@ -1,47 +0,0 @@ -import { LogLine, Stagehand } from "@/dist"; -import dotenv from "dotenv"; -import StagehandConfig from "@/stagehand.config"; -import chalk from "chalk"; - -// Load environment variables -dotenv.config(); - -const INSTRUCTION = - "Go to Google Japan and interact with it in Japanese. Tell me (in English) an authentic recipe that I can make with ingredients found in American grocery stores."; - -async function main() { - console.log(`\n${chalk.bold("Stagehand 🤘 Operator Example")}\n`); - - // Initialize Stagehand - const stagehand = new Stagehand({ - ...StagehandConfig, - logger: ({ level, message, timestamp }: LogLine) => { - console.log({ level, message, timestamp }); - }, - }); - - await stagehand.init(); - - try { - const agent = stagehand.agent(); - - // Execute the agent - console.log(`${chalk.cyan("↳")} Instruction: ${INSTRUCTION}`); - - const result = await agent.execute({ - instruction: INSTRUCTION, - maxSteps: 20, - }); - - console.log(`${chalk.green("✓")} Execution complete`); - console.log(`${chalk.yellow("⤷")} Result:`); - console.log(JSON.stringify(result, null, 2)); - console.log(chalk.white(result.message)); - } catch (error) { - console.log(`${chalk.red("✗")} Error: ${error}`); - } finally { - await stagehand.close(); - } -} - -main(); diff --git a/examples/parameterizeApiKey.ts b/examples/parameterizeApiKey.ts deleted file mode 100644 index 7af1c3eb1..000000000 --- a/examples/parameterizeApiKey.ts +++ /dev/null @@ -1,40 +0,0 @@ -import { Stagehand } from "@/dist"; -import { z } from "zod"; - -/** - * This example shows how to parameterize the API key for the LLM provider. - * - * In order to best demonstrate, unset the OPENAI_API_KEY environment variable and - * set the USE_OPENAI_API_KEY environment variable to your OpenAI API key. - * - * export USE_OPENAI_API_KEY=$OPENAI_API_KEY - * unset OPENAI_API_KEY - */ - -async function example() { - const stagehand = new Stagehand({ - env: "LOCAL", - verbose: 1, - enableCaching: false, - modelName: "gpt-4o", - modelClientOptions: { - apiKey: process.env.USE_OPENAI_API_KEY, - }, - }); - - await stagehand.init(); - await stagehand.page.goto("https://github.com/browserbase/stagehand"); - await stagehand.page.act({ action: "click on the contributors" }); - const contributor = await stagehand.page.extract({ - instruction: "extract the top contributor", - schema: z.object({ - username: z.string(), - url: z.string(), - }), - }); - console.log(`Our favorite contributor is ${contributor.username}`); -} - -(async () => { - await example(); -})(); diff --git a/examples/popup.ts b/examples/popup.ts deleted file mode 100644 index 72ec0369e..000000000 --- a/examples/popup.ts +++ /dev/null @@ -1,46 +0,0 @@ -/** - * This file is meant to be used as a scratchpad for developing new evals. - * To create a Stagehand project with best practices and configuration, run: - * - * npx create-browser-app@latest my-browser-app - */ - -import { ObserveResult, Stagehand } from "@/dist"; -import StagehandConfig from "@/stagehand.config"; - -async function example() { - const stagehand = new Stagehand(StagehandConfig); - await stagehand.init(); - - const page = await stagehand.page; - - let observePromise: Promise; - - page.on("popup", async (newPage) => { - observePromise = newPage.observe({ - instruction: "return all the next possible actions from the page", - }); - }); - - await page.goto( - "https://docs.browserbase.com/integrations/crew-ai/introduction", - ); - - await page.click( - "#content-area > div.relative.mt-8.prose.prose-gray.dark\\:prose-invert > p:nth-child(2) > a", - ); - - await page.waitForTimeout(5000); - - if (observePromise) { - const observeResult = await observePromise; - - console.log("Observed", observeResult.length, "actions"); - } - - await stagehand.close(); -} - -(async () => { - await example(); -})(); diff --git a/examples/try_wordle.ts b/examples/try_wordle.ts deleted file mode 100644 index a8f1fe22a..000000000 --- a/examples/try_wordle.ts +++ /dev/null @@ -1,24 +0,0 @@ -import { Stagehand } from "@/dist"; -import StagehandConfig from "@/stagehand.config"; - -async function example() { - const stagehand = new Stagehand({ - ...StagehandConfig, - }); - await stagehand.init(); - const page = stagehand.page; - await page.goto("https://www.nytimes.com/games/wordle/index.html"); - await page.act("click 'Continue'"); - await page.act("click 'Play'"); - await page.act("click cross sign on top right of 'How To Play' card"); - const word = "WORDS"; - for (const letter of word) { - await page.act(`press ${letter}`); - } - await page.act("press enter"); - await stagehand.close(); -} - -(async () => { - await example(); -})(); diff --git a/lib/StagehandContext.ts b/lib/StagehandContext.ts deleted file mode 100644 index 5d8bf4e66..000000000 --- a/lib/StagehandContext.ts +++ /dev/null @@ -1,125 +0,0 @@ -import type { - BrowserContext as PlaywrightContext, - Page as PlaywrightPage, -} from "@playwright/test"; -import { Stagehand } from "./index"; -import { StagehandPage } from "./StagehandPage"; -import { Page } from "../types/page"; -import { EnhancedContext } from "../types/context"; - -export class StagehandContext { - private readonly stagehand: Stagehand; - private readonly intContext: EnhancedContext; - private pageMap: WeakMap; - private activeStagehandPage: StagehandPage | null = null; - - private constructor(context: PlaywrightContext, stagehand: Stagehand) { - this.stagehand = stagehand; - this.pageMap = new WeakMap(); - - // Create proxy around the context - this.intContext = new Proxy(context, { - get: (target, prop) => { - if (prop === "newPage") { - return async (): Promise => { - const pwPage = await target.newPage(); - const stagehandPage = await this.createStagehandPage(pwPage); - // Set as active page when created - this.setActivePage(stagehandPage); - return stagehandPage.page; - }; - } - if (prop === "pages") { - return (): Page[] => { - const pwPages = target.pages(); - // Convert all pages to StagehandPages synchronously - return pwPages.map((pwPage: PlaywrightPage) => { - let stagehandPage = this.pageMap.get(pwPage); - if (!stagehandPage) { - // Create a new StagehandPage and store it in the map - stagehandPage = new StagehandPage( - pwPage, - this.stagehand, - this, - this.stagehand.llmClient, - this.stagehand.userProvidedInstructions, - this.stagehand.apiClient, - this.stagehand.waitForCaptchaSolves, - ); - this.pageMap.set(pwPage, stagehandPage); - } - return stagehandPage.page; - }); - }; - } - return target[prop as keyof PlaywrightContext]; - }, - }) as unknown as EnhancedContext; - } - - private async createStagehandPage( - page: PlaywrightPage, - ): Promise { - const stagehandPage = await new StagehandPage( - page, - this.stagehand, - this, - this.stagehand.llmClient, - this.stagehand.userProvidedInstructions, - this.stagehand.apiClient, - this.stagehand.waitForCaptchaSolves, - ).init(); - this.pageMap.set(page, stagehandPage); - return stagehandPage; - } - - static async init( - context: PlaywrightContext, - stagehand: Stagehand, - ): Promise { - const instance = new StagehandContext(context, stagehand); - - // Initialize existing pages - const existingPages = context.pages(); - for (const page of existingPages) { - const stagehandPage = await instance.createStagehandPage(page); - // Set the first page as active - if (!instance.activeStagehandPage) { - instance.setActivePage(stagehandPage); - } - } - - return instance; - } - - public get context(): EnhancedContext { - return this.intContext; - } - - public async getStagehandPage(page: PlaywrightPage): Promise { - let stagehandPage = this.pageMap.get(page); - if (!stagehandPage) { - stagehandPage = await this.createStagehandPage(page); - } - // Update active page when getting a page - this.setActivePage(stagehandPage); - return stagehandPage; - } - - public async getStagehandPages(): Promise { - const pwPages = this.intContext.pages(); - return Promise.all( - pwPages.map((page: PlaywrightPage) => this.getStagehandPage(page)), - ); - } - - public setActivePage(page: StagehandPage): void { - this.activeStagehandPage = page; - // Update the stagehand's active page reference - this.stagehand["setActivePage"](page); - } - - public getActivePage(): StagehandPage | null { - return this.activeStagehandPage; - } -} diff --git a/lib/StagehandPage.ts b/lib/StagehandPage.ts deleted file mode 100644 index 6d2aad423..000000000 --- a/lib/StagehandPage.ts +++ /dev/null @@ -1,829 +0,0 @@ -import { Browserbase } from "@browserbasehq/sdk"; -import type { CDPSession, Page as PlaywrightPage } from "@playwright/test"; -import { chromium } from "@playwright/test"; -import { z } from "zod"; -import { Page, defaultExtractSchema } from "../types/page"; -import { - ExtractOptions, - ExtractResult, - HistoryEntry, - ObserveOptions, - ObserveResult, -} from "../types/stagehand"; -import { StagehandAPI } from "./api"; -import { StagehandActHandler } from "./handlers/actHandler"; -import { StagehandExtractHandler } from "./handlers/extractHandler"; -import { StagehandObserveHandler } from "./handlers/observeHandler"; -import { ActOptions, ActResult, GotoOptions, Stagehand } from "./index"; -import { LLMClient } from "./llm/LLMClient"; -import { StagehandContext } from "./StagehandContext"; -import { EnhancedContext } from "../types/context"; -import { clearOverlays } from "./utils"; -import { - StagehandError, - StagehandNotInitializedError, - StagehandEnvironmentError, - CaptchaTimeoutError, - StagehandNotImplementedError, - BrowserbaseSessionNotFoundError, - MissingLLMConfigurationError, - HandlerNotInitializedError, - StagehandDefaultError, -} from "../types/stagehandErrors"; -import { StagehandAPIError } from "@/types/stagehandApiErrors"; - -export class StagehandPage { - private stagehand: Stagehand; - private intPage: Page; - private intContext: StagehandContext; - private actHandler: StagehandActHandler; - private extractHandler: StagehandExtractHandler; - private observeHandler: StagehandObserveHandler; - private llmClient: LLMClient; - private cdpClient: CDPSession | null = null; - private api: StagehandAPI; - private userProvidedInstructions?: string; - private waitForCaptchaSolves: boolean; - private initialized: boolean = false; - private _history: Array = []; - - public get history(): ReadonlyArray { - return Object.freeze([...this._history]); - } - - constructor( - page: PlaywrightPage, - stagehand: Stagehand, - context: StagehandContext, - llmClient: LLMClient, - userProvidedInstructions?: string, - api?: StagehandAPI, - waitForCaptchaSolves?: boolean, - ) { - // Create a proxy to intercept all method calls and property access - this.intPage = new Proxy(page, { - get: (target: PlaywrightPage, prop: keyof PlaywrightPage) => { - // Special handling for our enhanced methods before initialization - if ( - !this.initialized && - (prop === ("act" as keyof Page) || - prop === ("extract" as keyof Page) || - prop === ("observe" as keyof Page) || - prop === ("on" as keyof Page)) - ) { - return () => { - throw new StagehandNotInitializedError(String(prop)); - }; - } - - const value = target[prop]; - // If the property is a function, wrap it to update active page before execution - if (typeof value === "function" && prop !== "on") { - return (...args: unknown[]) => { - // Update active page before executing the method - this.intContext.setActivePage(this); - return value.apply(target, args); - }; - } - return value; - }, - }) as Page; - - this.stagehand = stagehand; - this.intContext = context; - this.llmClient = llmClient; - this.api = api; - this.userProvidedInstructions = userProvidedInstructions; - this.waitForCaptchaSolves = waitForCaptchaSolves ?? false; - - if (this.llmClient) { - this.actHandler = new StagehandActHandler({ - logger: this.stagehand.logger, - stagehandPage: this, - selfHeal: this.stagehand.selfHeal, - }); - this.extractHandler = new StagehandExtractHandler({ - stagehand: this.stagehand, - logger: this.stagehand.logger, - stagehandPage: this, - userProvidedInstructions, - }); - this.observeHandler = new StagehandObserveHandler({ - stagehand: this.stagehand, - logger: this.stagehand.logger, - stagehandPage: this, - userProvidedInstructions, - }); - } - } - - private async _refreshPageFromAPI() { - if (!this.api) return; - - const sessionId = this.stagehand.browserbaseSessionID; - if (!sessionId) { - throw new BrowserbaseSessionNotFoundError(); - } - - const browserbase = new Browserbase({ - apiKey: process.env.BROWSERBASE_API_KEY, - }); - - const sessionStatus = await browserbase.sessions.retrieve(sessionId); - - const connectUrl = sessionStatus.connectUrl; - const browser = await chromium.connectOverCDP(connectUrl); - const context = browser.contexts()[0]; - const newPage = context.pages()[0]; - - const newStagehandPage = await new StagehandPage( - newPage, - this.stagehand, - this.intContext, - this.llmClient, - this.userProvidedInstructions, - this.api, - ).init(); - - this.intPage = newStagehandPage.page; - - if (this.stagehand.debugDom) { - this.stagehand.log({ - category: "deprecation", - message: - "Warning: debugDom is not supported in this version of Stagehand", - level: 1, - }); - } - await this.intPage.waitForLoadState("domcontentloaded"); - await this._waitForSettledDom(); - } - - /** - * Waits for a captcha to be solved when using Browserbase environment. - * - * @param timeoutMs - Optional timeout in milliseconds. If provided, the promise will reject if the captcha solving hasn't started within the given time. - * @throws StagehandEnvironmentError if called in a LOCAL environment - * @throws CaptchaTimeoutError if the timeout is reached before captcha solving starts - * @returns Promise that resolves when the captcha is solved - */ - public async waitForCaptchaSolve(timeoutMs?: number) { - if (this.stagehand.env === "LOCAL") { - throw new StagehandEnvironmentError( - this.stagehand.env, - "BROWSERBASE", - "waitForCaptcha method", - ); - } - - this.stagehand.log({ - category: "captcha", - message: "Waiting for captcha", - level: 1, - }); - - return new Promise((resolve, reject) => { - let started = false; - let timeoutId: NodeJS.Timeout; - - if (timeoutMs) { - timeoutId = setTimeout(() => { - if (!started) { - reject(new CaptchaTimeoutError()); - } - }, timeoutMs); - } - - this.intPage.on("console", (msg) => { - if (msg.text() === "browserbase-solving-finished") { - this.stagehand.log({ - category: "captcha", - message: "Captcha solving finished", - level: 1, - }); - if (timeoutId) clearTimeout(timeoutId); - resolve(); - } else if (msg.text() === "browserbase-solving-started") { - started = true; - this.stagehand.log({ - category: "captcha", - message: "Captcha solving started", - level: 1, - }); - } - }); - }); - } - - async init(): Promise { - try { - const page = this.intPage; - const stagehand = this.stagehand; - - // Create a proxy that updates active page on method calls - const handler = { - get: (target: PlaywrightPage, prop: string | symbol) => { - const value = target[prop as keyof PlaywrightPage]; - - // Handle enhanced methods - if (prop === "act" || prop === "extract" || prop === "observe") { - if (!this.llmClient) { - return () => { - throw new MissingLLMConfigurationError(); - }; - } - - // Use type assertion to safely call the method with proper typing - type EnhancedMethod = ( - options: - | ActOptions - | ExtractOptions - | ObserveOptions, - ) => Promise< - ActResult | ExtractResult | ObserveResult[] - >; - - const method = this[prop as keyof StagehandPage] as EnhancedMethod; - return async (options: unknown) => { - this.intContext.setActivePage(this); - return method.call(this, options); - }; - } - - // Handle screenshots with CDP - if (prop === "screenshot" && this.stagehand.env === "BROWSERBASE") { - return async ( - options: { - type?: "png" | "jpeg"; - quality?: number; - fullPage?: boolean; - clip?: { x: number; y: number; width: number; height: number }; - omitBackground?: boolean; - } = {}, - ) => { - const cdpOptions: Record = { - format: options.type === "jpeg" ? "jpeg" : "png", - quality: options.quality, - clip: options.clip, - omitBackground: options.omitBackground, - fromSurface: true, - }; - - if (options.fullPage) { - cdpOptions.captureBeyondViewport = true; - } - - const data = await this.sendCDP<{ data: string }>( - "Page.captureScreenshot", - cdpOptions, - ); - - // Convert base64 to buffer - const buffer = Buffer.from(data.data, "base64"); - - return buffer; - }; - } - - // Handle goto specially - if (prop === "goto") { - return async (url: string, options: GotoOptions) => { - this.intContext.setActivePage(this); - const result = this.api - ? await this.api.goto(url, options) - : await target.goto(url, options); - - this.addToHistory("navigate", { url, options }, result); - - if (this.waitForCaptchaSolves) { - try { - await this.waitForCaptchaSolve(1000); - } catch { - // ignore - } - } - - if (this.api) { - await this._refreshPageFromAPI(); - } else { - if (stagehand.debugDom) { - this.stagehand.log({ - category: "deprecation", - message: - "Warning: debugDom is not supported in this version of Stagehand", - level: 1, - }); - } - await target.waitForLoadState("domcontentloaded"); - await this._waitForSettledDom(); - } - return result; - }; - } - - // Handle event listeners - if (prop === "on") { - return ( - event: keyof PlaywrightPage["on"], - listener: Parameters[1], - ) => { - if (event === "popup") { - return this.context.on("page", async (page: PlaywrightPage) => { - const newContext = await StagehandContext.init( - page.context(), - stagehand, - ); - const newStagehandPage = new StagehandPage( - page, - stagehand, - newContext, - this.llmClient, - ); - - await newStagehandPage.init(); - listener(newStagehandPage.page); - }); - } - this.intContext.setActivePage(this); - return target.on(event, listener); - }; - } - - // For all other method calls, update active page - if (typeof value === "function") { - return (...args: unknown[]) => { - this.intContext.setActivePage(this); - return value.apply(target, args); - }; - } - - return value; - }, - }; - - this.intPage = new Proxy(page, handler) as unknown as Page; - this.initialized = true; - return this; - } catch (err: unknown) { - if (err instanceof StagehandError || err instanceof StagehandAPIError) { - throw err; - } - throw new StagehandDefaultError(err); - } - } - - public get page(): Page { - return this.intPage; - } - - public get context(): EnhancedContext { - return this.intContext.context; - } - - // We can make methods public because StagehandPage is private to the Stagehand class. - // When a user gets stagehand.page, they are getting a proxy to the Playwright page. - // We can override the methods on the proxy to add our own behavior - public async _waitForSettledDom(timeoutMs?: number) { - try { - const timeout = timeoutMs ?? this.stagehand.domSettleTimeoutMs; - let timeoutHandle: NodeJS.Timeout; - - await this.page.waitForLoadState("domcontentloaded"); - - const timeoutPromise = new Promise((resolve) => { - timeoutHandle = setTimeout(() => { - this.stagehand.log({ - category: "dom", - message: "DOM settle timeout exceeded, continuing anyway", - level: 1, - auxiliary: { - timeout_ms: { - value: timeout.toString(), - type: "integer", - }, - }, - }); - resolve(); - }, timeout); - }); - - try { - await Promise.race([ - this.page.evaluate(() => { - return new Promise((resolve) => { - if (typeof window.waitForDomSettle === "function") { - window.waitForDomSettle().then(resolve); - } else { - console.warn( - "waitForDomSettle is not defined, considering DOM as settled", - ); - resolve(); - } - }); - }), - this.page.waitForLoadState("domcontentloaded"), - this.page.waitForSelector("body"), - timeoutPromise, - ]); - } finally { - clearTimeout(timeoutHandle!); - } - } catch (e) { - this.stagehand.log({ - category: "dom", - message: "Error in waitForSettledDom", - level: 1, - auxiliary: { - error: { - value: e.message, - type: "string", - }, - trace: { - value: e.stack, - type: "string", - }, - }, - }); - } - } - - public addToHistory( - method: HistoryEntry["method"], - parameters: - | ActOptions - | ExtractOptions - | ObserveOptions - | { url: string; options: GotoOptions } - | string, - result?: unknown, - ): void { - this._history.push({ - method, - parameters, - result: result ?? null, - timestamp: new Date().toISOString(), - }); - } - - async act( - actionOrOptions: string | ActOptions | ObserveResult, - ): Promise { - try { - if (!this.actHandler) { - throw new HandlerNotInitializedError("Act"); - } - - await clearOverlays(this.page); - - // If actionOrOptions is an ObserveResult, we call actFromObserveResult. - // We need to ensure there is both a selector and a method in the ObserveResult. - if (typeof actionOrOptions === "object" && actionOrOptions !== null) { - // If it has selector AND method => treat as ObserveResult - if ("selector" in actionOrOptions && "method" in actionOrOptions) { - const observeResult = actionOrOptions as ObserveResult; - - if (this.api) { - const result = await this.api.act(observeResult); - await this._refreshPageFromAPI(); - this.addToHistory("act", observeResult, result); - return result; - } - - // validate observeResult.method, etc. - return this.actHandler.actFromObserveResult(observeResult); - } else { - // If it's an object but no selector/method, - // check that it's truly ActOptions (i.e., has an `action` field). - if (!("action" in actionOrOptions)) { - throw new StagehandError( - "Invalid argument. Valid arguments are: a string, an ActOptions object, " + - "or an ObserveResult WITH 'selector' and 'method' fields.", - ); - } - } - } else if (typeof actionOrOptions === "string") { - // Convert string to ActOptions - actionOrOptions = { action: actionOrOptions }; - } else { - throw new StagehandError( - "Invalid argument: you may have called act with an empty ObserveResult.\n" + - "Valid arguments are: a string, an ActOptions object, or an ObserveResult " + - "WITH 'selector' and 'method' fields.", - ); - } - - const { action, modelName, modelClientOptions } = actionOrOptions; - - if (this.api) { - const result = await this.api.act(actionOrOptions); - await this._refreshPageFromAPI(); - this.addToHistory("act", actionOrOptions, result); - return result; - } - - const requestId = Math.random().toString(36).substring(2); - const llmClient: LLMClient = modelName - ? this.stagehand.llmProvider.getClient(modelName, modelClientOptions) - : this.llmClient; - - this.stagehand.log({ - category: "act", - message: "running act", - level: 1, - auxiliary: { - action: { - value: action, - type: "string", - }, - requestId: { - value: requestId, - type: "string", - }, - modelName: { - value: llmClient.modelName, - type: "string", - }, - }, - }); - - const result = await this.actHandler.observeAct( - actionOrOptions, - this.observeHandler, - llmClient, - requestId, - ); - - this.addToHistory("act", actionOrOptions, result); - return result; - } catch (err: unknown) { - if (err instanceof StagehandError || err instanceof StagehandAPIError) { - throw err; - } - throw new StagehandDefaultError(err); - } - } - - async extract( - instructionOrOptions?: string | ExtractOptions, - ): Promise> { - try { - if (!this.extractHandler) { - throw new HandlerNotInitializedError("Extract"); - } - - await clearOverlays(this.page); - - // check if user called extract() with no arguments - if (!instructionOrOptions) { - let result: ExtractResult; - if (this.api) { - result = await this.api.extract({}); - } else { - result = await this.extractHandler.extract(); - } - this.addToHistory("extract", instructionOrOptions, result); - return result; - } - - const options: ExtractOptions = - typeof instructionOrOptions === "string" - ? { - instruction: instructionOrOptions, - schema: defaultExtractSchema as T, - } - : instructionOrOptions; - - const { - instruction, - schema, - modelName, - modelClientOptions, - domSettleTimeoutMs, - useTextExtract, - selector, - } = options; - - // Throw a NotImplementedError if the user passed in an `xpath` - // and `useTextExtract` is false - if (selector && useTextExtract !== true) { - throw new StagehandNotImplementedError( - "Passing an xpath into extract is only supported when `useTextExtract: true`.", - ); - } - - if (this.api) { - const result = await this.api.extract(options); - this.addToHistory("extract", instructionOrOptions, result); - return result; - } - - const requestId = Math.random().toString(36).substring(2); - const llmClient = modelName - ? this.stagehand.llmProvider.getClient(modelName, modelClientOptions) - : this.llmClient; - - this.stagehand.log({ - category: "extract", - message: "running extract", - level: 1, - auxiliary: { - instruction: { - value: instruction, - type: "string", - }, - requestId: { - value: requestId, - type: "string", - }, - modelName: { - value: llmClient.modelName, - type: "string", - }, - }, - }); - - const result = await this.extractHandler - .extract({ - instruction, - schema, - llmClient, - requestId, - domSettleTimeoutMs, - useTextExtract, - selector, - }) - .catch((e) => { - this.stagehand.log({ - category: "extract", - message: "error extracting", - level: 1, - auxiliary: { - error: { - value: e.message, - type: "string", - }, - trace: { - value: e.stack, - type: "string", - }, - }, - }); - - if (this.stagehand.enableCaching) { - this.stagehand.llmProvider.cleanRequestCache(requestId); - } - - throw e; - }); - - this.addToHistory("extract", instructionOrOptions, result); - - return result; - } catch (err: unknown) { - if (err instanceof StagehandError || err instanceof StagehandAPIError) { - throw err; - } - throw new StagehandDefaultError(err); - } - } - - async observe( - instructionOrOptions?: string | ObserveOptions, - ): Promise { - try { - if (!this.observeHandler) { - throw new HandlerNotInitializedError("Observe"); - } - - await clearOverlays(this.page); - - const options: ObserveOptions = - typeof instructionOrOptions === "string" - ? { instruction: instructionOrOptions } - : instructionOrOptions || {}; - - const { - instruction, - modelName, - modelClientOptions, - domSettleTimeoutMs, - returnAction = true, - onlyVisible = false, - drawOverlay, - } = options; - - if (this.api) { - const result = await this.api.observe(options); - this.addToHistory("observe", instructionOrOptions, result); - return result; - } - - const requestId = Math.random().toString(36).substring(2); - const llmClient = modelName - ? this.stagehand.llmProvider.getClient(modelName, modelClientOptions) - : this.llmClient; - - this.stagehand.log({ - category: "observe", - message: "running observe", - level: 1, - auxiliary: { - instruction: { - value: instruction, - type: "string", - }, - requestId: { - value: requestId, - type: "string", - }, - modelName: { - value: llmClient.modelName, - type: "string", - }, - onlyVisible: { - value: onlyVisible ? "true" : "false", - type: "boolean", - }, - }, - }); - - const result = await this.observeHandler - .observe({ - instruction, - llmClient, - requestId, - domSettleTimeoutMs, - returnAction, - onlyVisible, - drawOverlay, - }) - .catch((e) => { - this.stagehand.log({ - category: "observe", - message: "error observing", - level: 1, - auxiliary: { - error: { - value: e.message, - type: "string", - }, - trace: { - value: e.stack, - type: "string", - }, - requestId: { - value: requestId, - type: "string", - }, - instruction: { - value: instruction, - type: "string", - }, - }, - }); - - if (this.stagehand.enableCaching) { - this.stagehand.llmProvider.cleanRequestCache(requestId); - } - - throw e; - }); - - this.addToHistory("observe", instructionOrOptions, result); - - return result; - } catch (err: unknown) { - if (err instanceof StagehandError || err instanceof StagehandAPIError) { - throw err; - } - throw new StagehandDefaultError(err); - } - } - - async getCDPClient(): Promise { - if (!this.cdpClient) { - this.cdpClient = await this.context.newCDPSession(this.page); - } - return this.cdpClient; - } - - async sendCDP( - command: string, - args?: Record, - ): Promise { - const client = await this.getCDPClient(); - // Type assertion needed because CDP command strings are not fully typed - return client.send( - command as Parameters[0], - args || {}, - ) as Promise; - } - - async enableCDP(domain: string): Promise { - await this.sendCDP(`${domain}.enable`, {}); - } - - async disableCDP(domain: string): Promise { - await this.sendCDP(`${domain}.disable`, {}); - } -} diff --git a/lib/a11y/utils.ts b/lib/a11y/utils.ts deleted file mode 100644 index f19ba765a..000000000 --- a/lib/a11y/utils.ts +++ /dev/null @@ -1,825 +0,0 @@ -import { AccessibilityNode, TreeResult, AXNode } from "../../types/context"; -import { StagehandPage } from "../StagehandPage"; -import { LogLine } from "../../types/log"; -import { CDPSession, Page, Locator } from "playwright"; -import { - PlaywrightCommandMethodNotSupportedException, - PlaywrightCommandException, -} from "@/types/playwright"; - -// Parser function for str output -export function formatSimplifiedTree( - node: AccessibilityNode, - level = 0, -): string { - const indent = " ".repeat(level); - let result = `${indent}[${node.nodeId}] ${node.role}${ - node.name ? `: ${node.name}` : "" - }\n`; - - if (node.children?.length) { - result += node.children - .map((child) => formatSimplifiedTree(child, level + 1)) - .join(""); - } - return result; -} - -/** - * Helper function to remove or collapse unnecessary structural nodes - * Handles three cases: - * 1. Removes generic/none nodes with no children - * 2. Collapses generic/none nodes with single child - * 3. Keeps generic/none nodes with multiple children but cleans their subtrees - * and attempts to resolve their role to a DOM tag name - */ -async function cleanStructuralNodes( - node: AccessibilityNode, - page?: StagehandPage, - logger?: (logLine: LogLine) => void, -): Promise { - // 1) Filter out nodes with negative IDs - if (node.nodeId && parseInt(node.nodeId) < 0) { - return null; - } - - // 2) Base case: if no children exist, this is effectively a leaf. - // If it's "generic" or "none", we remove it; otherwise, keep it. - if (!node.children || node.children.length === 0) { - return node.role === "generic" || node.role === "none" ? null : node; - } - - // 3) Recursively clean children - const cleanedChildrenPromises = node.children.map((child) => - cleanStructuralNodes(child, page, logger), - ); - const resolvedChildren = await Promise.all(cleanedChildrenPromises); - let cleanedChildren = resolvedChildren.filter( - (child): child is AccessibilityNode => child !== null, - ); - - // 4) **Prune** "generic" or "none" nodes first, - // before resolving them to their tag names. - if (node.role === "generic" || node.role === "none") { - if (cleanedChildren.length === 1) { - // Collapse single-child structural node - return cleanedChildren[0]; - } else if (cleanedChildren.length === 0) { - // Remove empty structural node - return null; - } - // If we have multiple children, we keep this node as a container. - // We'll update role below if needed. - } - - // 5) If we still have a "generic"/"none" node after pruning - // (i.e., because it had multiple children), now we try - // to resolve and replace its role with the DOM tag name. - if ( - page && - logger && - node.backendDOMNodeId !== undefined && - (node.role === "generic" || node.role === "none") - ) { - try { - const { object } = await page.sendCDP<{ - object: { objectId?: string }; - }>("DOM.resolveNode", { - backendNodeId: node.backendDOMNodeId, - }); - - if (object && object.objectId) { - try { - // Get the tagName for the node - const { result } = await page.sendCDP<{ - result: { type: string; value?: string }; - }>("Runtime.callFunctionOn", { - objectId: object.objectId, - functionDeclaration: ` - function() { - return this.tagName ? this.tagName.toLowerCase() : ""; - } - `, - returnByValue: true, - }); - - // If we got a tagName, update the node's role - if (result?.value) { - node.role = result.value; - } - } catch (tagNameError) { - logger({ - category: "observation", - message: `Could not fetch tagName for node ${node.backendDOMNodeId}`, - level: 2, - auxiliary: { - error: { - value: tagNameError.message, - type: "string", - }, - }, - }); - } - } - } catch (resolveError) { - logger({ - category: "observation", - message: `Could not resolve DOM node ID ${node.backendDOMNodeId}`, - level: 2, - auxiliary: { - error: { - value: resolveError.message, - type: "string", - }, - }, - }); - } - } - - // rm redundant StaticText children - cleanedChildren = removeRedundantStaticTextChildren(node, cleanedChildren); - - if (cleanedChildren.length === 0) { - if (node.role === "generic" || node.role === "none") { - return null; - } else { - return { ...node, children: [] }; - } - } - - // 6) Return the updated node. - // If it has children, update them; otherwise keep it as-is. - return cleanedChildren.length > 0 - ? { ...node, children: cleanedChildren } - : node; -} - -/** - * Builds a hierarchical tree structure from a flat array of accessibility nodes. - * The function processes nodes in multiple passes to create a clean, meaningful tree. - * @param nodes - Flat array of accessibility nodes from the CDP - * @returns Object containing both the tree structure and a simplified string representation - */ -export async function buildHierarchicalTree( - nodes: AccessibilityNode[], - page?: StagehandPage, - logger?: (logLine: LogLine) => void, -): Promise { - // Map to store nodeId -> URL for only those nodes that do have a URL. - const idToUrl: Record = {}; - - // Map to store processed nodes for quick lookup - const nodeMap = new Map(); - const iframe_list: AccessibilityNode[] = []; - - // First pass: Create nodes that are meaningful - // We only keep nodes that either have a name or children to avoid cluttering the tree - nodes.forEach((node) => { - // Skip node if its ID is negative (e.g., "-1000002014") - const nodeIdValue = parseInt(node.nodeId, 10); - if (nodeIdValue < 0) { - return; - } - - const url = extractUrlFromAXNode(node); - if (url) { - idToUrl[node.nodeId] = url; - } - - const hasChildren = node.childIds && node.childIds.length > 0; - const hasValidName = node.name && node.name.trim() !== ""; - const isInteractive = - node.role !== "none" && - node.role !== "generic" && - node.role !== "InlineTextBox"; //add other interactive roles here - - // Include nodes that are either named, have children, or are interactive - if (!hasValidName && !hasChildren && !isInteractive) { - return; - } - - // Create a clean node object with only relevant properties - nodeMap.set(node.nodeId, { - role: node.role, - nodeId: node.nodeId, - ...(hasValidName && { name: node.name }), // Only include name if it exists and isn't empty - ...(node.description && { description: node.description }), - ...(node.value && { value: node.value }), - ...(node.backendDOMNodeId !== undefined && { - backendDOMNodeId: node.backendDOMNodeId, - }), - }); - }); - - // Second pass: Establish parent-child relationships - // This creates the actual tree structure by connecting nodes based on parentId - nodes.forEach((node) => { - // Add iframes to a list and include in the return object - const isIframe = node.role === "Iframe"; - if (isIframe) { - const iframeNode = { - role: node.role, - nodeId: node.nodeId, - }; - iframe_list.push(iframeNode); - } - if (node.parentId && nodeMap.has(node.nodeId)) { - const parentNode = nodeMap.get(node.parentId); - const currentNode = nodeMap.get(node.nodeId); - - if (parentNode && currentNode) { - if (!parentNode.children) { - parentNode.children = []; - } - parentNode.children.push(currentNode); - } - } - }); - - // Final pass: Build the root-level tree and clean up structural nodes - const rootNodes = nodes - .filter((node) => !node.parentId && nodeMap.has(node.nodeId)) // Get root nodes - .map((node) => nodeMap.get(node.nodeId)) - .filter(Boolean) as AccessibilityNode[]; - - const cleanedTreePromises = rootNodes.map((node) => - cleanStructuralNodes(node, page, logger), - ); - const finalTree = (await Promise.all(cleanedTreePromises)).filter( - Boolean, - ) as AccessibilityNode[]; - - // Generate a simplified string representation of the tree - const simplifiedFormat = finalTree - .map((node) => formatSimplifiedTree(node)) - .join("\n"); - - return { - tree: finalTree, - simplified: simplifiedFormat, - iframes: iframe_list, - idToUrl: idToUrl, - }; -} - -/** - * Retrieves the full accessibility tree via CDP and transforms it into a hierarchical structure. - */ -export async function getAccessibilityTree( - page: StagehandPage, - logger: (logLine: LogLine) => void, -): Promise { - await page.enableCDP("Accessibility"); - - try { - // Identify which elements are scrollable and get their backendNodeIds - const scrollableBackendIds = await findScrollableElementIds(page); - - // Fetch the full accessibility tree from Chrome DevTools Protocol - const { nodes } = await page.sendCDP<{ nodes: AXNode[] }>( - "Accessibility.getFullAXTree", - ); - const startTime = Date.now(); - - // Transform into hierarchical structure - const hierarchicalTree = await buildHierarchicalTree( - nodes.map((node) => { - let roleValue = node.role?.value || ""; - - if (scrollableBackendIds.has(node.backendDOMNodeId)) { - if (roleValue === "generic" || roleValue === "none") { - roleValue = "scrollable"; - } else { - roleValue = roleValue ? `scrollable, ${roleValue}` : "scrollable"; - } - } - - return { - role: roleValue, - name: node.name?.value, - description: node.description?.value, - value: node.value?.value, - nodeId: node.nodeId, - backendDOMNodeId: node.backendDOMNodeId, - parentId: node.parentId, - childIds: node.childIds, - properties: node.properties, - }; - }), - page, - logger, - ); - - logger({ - category: "observation", - message: `got accessibility tree in ${Date.now() - startTime}ms`, - level: 1, - }); - return hierarchicalTree; - } catch (error) { - logger({ - category: "observation", - message: "Error getting accessibility tree", - level: 1, - auxiliary: { - error: { - value: error.message, - type: "string", - }, - trace: { - value: error.stack, - type: "string", - }, - }, - }); - throw error; - } finally { - await page.disableCDP("Accessibility"); - } -} - -// This function is wrapped into a string and sent as a CDP command -// It is not meant to be actually executed here -const functionString = ` -function getNodePath(el) { - if (!el || (el.nodeType !== Node.ELEMENT_NODE && el.nodeType !== Node.TEXT_NODE)) { - console.log("el is not a valid node type"); - return ""; - } - - const parts = []; - let current = el; - - while (current && (current.nodeType === Node.ELEMENT_NODE || current.nodeType === Node.TEXT_NODE)) { - let index = 0; - let hasSameTypeSiblings = false; - const siblings = current.parentElement - ? Array.from(current.parentElement.childNodes) - : []; - - for (let i = 0; i < siblings.length; i++) { - const sibling = siblings[i]; - if ( - sibling.nodeType === current.nodeType && - sibling.nodeName === current.nodeName - ) { - index = index + 1; - hasSameTypeSiblings = true; - if (sibling.isSameNode(current)) { - break; - } - } - } - - if (!current || !current.parentNode) break; - if (current.nodeName.toLowerCase() === "html"){ - parts.unshift("html"); - break; - } - - // text nodes are handled differently in XPath - if (current.nodeName !== "#text") { - const tagName = current.nodeName.toLowerCase(); - const pathIndex = hasSameTypeSiblings ? \`[\${index}]\` : ""; - parts.unshift(\`\${tagName}\${pathIndex}\`); - } - - current = current.parentElement; - } - - return parts.length ? \`/\${parts.join("/")}\` : ""; -}`; - -export async function getXPathByResolvedObjectId( - cdpClient: CDPSession, - resolvedObjectId: string, -): Promise { - const { result } = await cdpClient.send("Runtime.callFunctionOn", { - objectId: resolvedObjectId, - functionDeclaration: `function() { - ${functionString} - return getNodePath(this); - }`, - returnByValue: true, - }); - - return result.value || ""; -} - -/** - * `findScrollableElementIds` is a function that identifies elements in - * the browser that are deemed "scrollable". At a high level, it does the - * following: - * - Calls the browser-side `window.getScrollableElementXpaths()` function, - * which returns a list of XPaths for scrollable containers. - * - Iterates over the returned list of XPaths, locating each element in the DOM - * using `stagehandPage.sendCDP(...)` - * - During each iteration, we call `Runtime.evaluate` to run `document.evaluate(...)` - * with each XPath, obtaining a `RemoteObject` reference if it exists. - * - Then, for each valid object reference, we call `DOM.describeNode` to retrieve - * the element’s `backendNodeId`. - * - Collects all resulting `backendNodeId`s in a Set and returns them. - * - * @param stagehandPage - A StagehandPage instance with built-in CDP helpers. - * @returns A Promise that resolves to a Set of unique `backendNodeId`s corresponding - * to scrollable elements in the DOM. - */ -export async function findScrollableElementIds( - stagehandPage: StagehandPage, -): Promise> { - // get the xpaths of the scrollable elements - const xpaths = await stagehandPage.page.evaluate(() => { - return window.getScrollableElementXpaths(); - }); - - const scrollableBackendIds = new Set(); - - for (const xpath of xpaths) { - if (!xpath) continue; - - // evaluate the XPath in the stagehandPage - const { result } = await stagehandPage.sendCDP<{ - result?: { objectId?: string }; - }>("Runtime.evaluate", { - expression: ` - (function() { - const res = document.evaluate(${JSON.stringify( - xpath, - )}, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null); - return res.singleNodeValue; - })(); - `, - returnByValue: false, - }); - - // if we have an objectId, call DOM.describeNode to get backendNodeId - if (result?.objectId) { - const { node } = await stagehandPage.sendCDP<{ - node?: { backendNodeId?: number }; - }>("DOM.describeNode", { - objectId: result.objectId, - }); - - if (node?.backendNodeId) { - scrollableBackendIds.add(node.backendNodeId); - } - } - } - - return scrollableBackendIds; -} - -/** - * Removes any StaticText children whose combined text equals the parent's name. - * This is most often used to avoid duplicating a link's accessible name in separate child nodes. - * - * @param parent The parent accessibility node whose `.name` we check. - * @param children The parent's current children list, typically after cleaning. - * @returns A filtered list of children with redundant StaticText nodes removed. - */ -function removeRedundantStaticTextChildren( - parent: AccessibilityNode, - children: AccessibilityNode[], -): AccessibilityNode[] { - if (!parent.name) { - return children; - } - - const parentName = parent.name.replace(/\s+/g, " ").trim(); - - // Gather all StaticText children and combine their text - const staticTextChildren = children.filter( - (child) => child.role === "StaticText" && child.name, - ); - const combinedChildText = staticTextChildren - .map((child) => child.name!.replace(/\s+/g, " ").trim()) - .join(""); - - // If the combined text exactly matches the parent's name, remove those child nodes - if (combinedChildText === parentName) { - return children.filter((child) => child.role !== "StaticText"); - } - - return children; -} - -function extractUrlFromAXNode(axNode: AccessibilityNode): string | undefined { - if (!axNode.properties) return undefined; - const urlProp = axNode.properties.find((prop) => prop.name === "url"); - if (urlProp && urlProp.value && typeof urlProp.value.value === "string") { - return urlProp.value.value.trim(); - } - return undefined; -} - -export async function performPlaywrightMethod( - stagehandPage: Page, - logger: (logLine: LogLine) => void, - method: string, - args: unknown[], - xpath: string, -) { - const locator = stagehandPage.locator(`xpath=${xpath}`).first(); - const initialUrl = stagehandPage.url(); - - logger({ - category: "action", - message: "performing playwright method", - level: 2, - auxiliary: { - xpath: { - value: xpath, - type: "string", - }, - method: { - value: method, - type: "string", - }, - }, - }); - - if (method === "scrollIntoView") { - logger({ - category: "action", - message: "scrolling element into view", - level: 2, - auxiliary: { - xpath: { - value: xpath, - type: "string", - }, - }, - }); - try { - await locator - .evaluate((element: HTMLElement) => { - element.scrollIntoView({ behavior: "smooth", block: "center" }); - }) - .catch((e: Error) => { - logger({ - category: "action", - message: "error scrolling element into view", - level: 1, - auxiliary: { - error: { - value: e.message, - type: "string", - }, - trace: { - value: e.stack, - type: "string", - }, - xpath: { - value: xpath, - type: "string", - }, - }, - }); - }); - } catch (e) { - logger({ - category: "action", - message: "error scrolling element into view", - level: 1, - auxiliary: { - error: { - value: e.message, - type: "string", - }, - trace: { - value: e.stack, - type: "string", - }, - xpath: { - value: xpath, - type: "string", - }, - }, - }); - - throw new PlaywrightCommandException(e.message); - } - } else if (method === "fill" || method === "type") { - try { - await locator.fill(""); - await locator.click(); - const text = args[0]?.toString(); - for (const char of text) { - await stagehandPage.keyboard.type(char, { - delay: Math.random() * 50 + 25, - }); - } - } catch (e) { - logger({ - category: "action", - message: "error filling element", - level: 1, - auxiliary: { - error: { - value: e.message, - type: "string", - }, - trace: { - value: e.stack, - type: "string", - }, - xpath: { - value: xpath, - type: "string", - }, - }, - }); - - throw new PlaywrightCommandException(e.message); - } - } else if (method === "press") { - try { - const key = args[0]?.toString(); - await stagehandPage.keyboard.press(key); - } catch (e) { - logger({ - category: "action", - message: "error pressing key", - level: 1, - auxiliary: { - error: { - value: e.message, - type: "string", - }, - trace: { - value: e.stack, - type: "string", - }, - key: { - value: args[0]?.toString() ?? "unknown", - type: "string", - }, - }, - }); - - throw new PlaywrightCommandException(e.message); - } - } else if (typeof locator[method as keyof typeof locator] === "function") { - // Log current URL before action - logger({ - category: "action", - message: "page URL before action", - level: 2, - auxiliary: { - url: { - value: stagehandPage.url(), - type: "string", - }, - }, - }); - - // Perform the action - try { - await ( - locator[method as keyof Locator] as unknown as ( - ...args: string[] - ) => Promise - )(...args.map((arg) => arg?.toString() || "")); - } catch (e) { - logger({ - category: "action", - message: "error performing method", - level: 1, - auxiliary: { - error: { - value: e.message, - type: "string", - }, - trace: { - value: e.stack, - type: "string", - }, - xpath: { - value: xpath, - type: "string", - }, - method: { - value: method, - type: "string", - }, - args: { - value: JSON.stringify(args), - type: "object", - }, - }, - }); - - throw new PlaywrightCommandException(e.message); - } - - // Handle navigation if a new page is opened - if (method === "click") { - logger({ - category: "action", - message: "clicking element, checking for page navigation", - level: 1, - auxiliary: { - xpath: { - value: xpath, - type: "string", - }, - }, - }); - - const newOpenedTab = await Promise.race([ - new Promise((resolve) => { - Promise.resolve(stagehandPage.context()).then((context) => { - context.once("page", (page: Page) => resolve(page)); - setTimeout(() => resolve(null), 1_500); - }); - }), - ]); - - logger({ - category: "action", - message: "clicked element", - level: 1, - auxiliary: { - newOpenedTab: { - value: newOpenedTab ? "opened a new tab" : "no new tabs opened", - type: "string", - }, - }, - }); - - if (newOpenedTab) { - logger({ - category: "action", - message: "new page detected (new tab) with URL", - level: 1, - auxiliary: { - url: { - value: newOpenedTab.url(), - type: "string", - }, - }, - }); - await newOpenedTab.close(); - await stagehandPage.goto(newOpenedTab.url()); - await stagehandPage.waitForLoadState("domcontentloaded"); - } - - await Promise.race([ - stagehandPage.waitForLoadState("networkidle"), - new Promise((resolve) => setTimeout(resolve, 5_000)), - ]).catch((e) => { - logger({ - category: "action", - message: "network idle timeout hit", - level: 1, - auxiliary: { - trace: { - value: e.stack, - type: "string", - }, - message: { - value: e.message, - type: "string", - }, - }, - }); - }); - - logger({ - category: "action", - message: "finished waiting for (possible) page navigation", - level: 1, - }); - - if (stagehandPage.url() !== initialUrl) { - logger({ - category: "action", - message: "new page detected with URL", - level: 1, - auxiliary: { - url: { - value: stagehandPage.url(), - type: "string", - }, - }, - }); - } - } - } else { - logger({ - category: "action", - message: "chosen method is invalid", - level: 1, - auxiliary: { - method: { - value: method, - type: "string", - }, - }, - }); - - throw new PlaywrightCommandMethodNotSupportedException( - `Method ${method} not supported`, - ); - } -} diff --git a/lib/agent/AgentClient.ts b/lib/agent/AgentClient.ts deleted file mode 100644 index ac0aa83af..000000000 --- a/lib/agent/AgentClient.ts +++ /dev/null @@ -1,44 +0,0 @@ -import { - AgentAction, - AgentResult, - AgentType, - AgentExecutionOptions, -} from "@/types/agent"; - -/** - * Abstract base class for agent clients - * This provides a common interface for all agent implementations - */ -export abstract class AgentClient { - public type: AgentType; - public modelName: string; - public clientOptions: Record; - public userProvidedInstructions?: string; - - constructor( - type: AgentType, - modelName: string, - userProvidedInstructions?: string, - ) { - this.type = type; - this.modelName = modelName; - this.userProvidedInstructions = userProvidedInstructions; - this.clientOptions = {}; - } - - abstract execute(options: AgentExecutionOptions): Promise; - - abstract captureScreenshot( - options?: Record, - ): Promise; - - abstract setViewport(width: number, height: number): void; - - abstract setCurrentUrl(url: string): void; - - abstract setScreenshotProvider(provider: () => Promise): void; - - abstract setActionHandler( - handler: (action: AgentAction) => Promise, - ): void; -} diff --git a/lib/agent/AgentProvider.ts b/lib/agent/AgentProvider.ts deleted file mode 100644 index cc8110fa4..000000000 --- a/lib/agent/AgentProvider.ts +++ /dev/null @@ -1,90 +0,0 @@ -import { LogLine } from "@/types/log"; -import { AgentClient } from "./AgentClient"; -import { AgentType } from "@/types/agent"; -import { OpenAICUAClient } from "./OpenAICUAClient"; -import { AnthropicCUAClient } from "./AnthropicCUAClient"; -import { - UnsupportedModelError, - UnsupportedModelProviderError, -} from "@/types/stagehandErrors"; - -// Map model names to their provider types -const modelToAgentProviderMap: Record = { - "computer-use-preview": "openai", - "claude-3-5-sonnet-20240620": "anthropic", - "claude-3-7-sonnet-20250219": "anthropic", // Add newer Claude models -}; - -/** - * Provider for agent clients - * This class is responsible for creating the appropriate agent client - * based on the provider type - */ -export class AgentProvider { - private logger: (message: LogLine) => void; - - /** - * Create a new agent provider - */ - constructor(logger: (message: LogLine) => void) { - this.logger = logger; - } - - getClient( - modelName: string, - clientOptions?: Record, - userProvidedInstructions?: string, - ): AgentClient { - const type = AgentProvider.getAgentProvider(modelName); - this.logger({ - category: "agent", - message: `Getting agent client for type: ${type}, model: ${modelName}`, - level: 2, - }); - - try { - switch (type) { - case "openai": - return new OpenAICUAClient( - type, - modelName, - userProvidedInstructions, - clientOptions, - ); - case "anthropic": - return new AnthropicCUAClient( - type, - modelName, - userProvidedInstructions, - clientOptions, - ); - default: - throw new UnsupportedModelProviderError( - ["openai", "anthropic"], - "Computer Use Agent", - ); - } - } catch (error) { - const errorMessage = - error instanceof Error ? error.message : String(error); - this.logger({ - category: "agent", - message: `Error creating agent client: ${errorMessage}`, - level: 0, - }); - throw error; - } - } - - static getAgentProvider(modelName: string): AgentType { - // First check the exact model name in the map - if (modelName in modelToAgentProviderMap) { - return modelToAgentProviderMap[modelName]; - } - - throw new UnsupportedModelError( - Object.keys(modelToAgentProviderMap), - "Computer Use Agent", - ); - } -} diff --git a/lib/agent/AnthropicCUAClient.ts b/lib/agent/AnthropicCUAClient.ts deleted file mode 100644 index fbbc97d82..000000000 --- a/lib/agent/AnthropicCUAClient.ts +++ /dev/null @@ -1,862 +0,0 @@ -import Anthropic from "@anthropic-ai/sdk"; -import { LogLine } from "@/types/log"; -import { - AgentAction, - AgentResult, - AgentType, - AgentExecutionOptions, - ToolUseItem, - AnthropicMessage, - AnthropicContentBlock, - AnthropicTextBlock, - AnthropicToolResult, -} from "@/types/agent"; -import { AgentClient } from "./AgentClient"; -import { AgentScreenshotProviderError } from "@/types/stagehandErrors"; - -export type ResponseInputItem = AnthropicMessage | AnthropicToolResult; - -/** - * Client for Anthropic's Computer Use API - * This implementation uses the official Anthropic Messages API for Computer Use - */ -export class AnthropicCUAClient extends AgentClient { - private apiKey: string; - private baseURL?: string; - private client: Anthropic; - public lastMessageId?: string; - private currentViewport = { width: 1024, height: 768 }; - private currentUrl?: string; - private screenshotProvider?: () => Promise; - private actionHandler?: (action: AgentAction) => Promise; - private thinkingBudget: number | null = null; - - constructor( - type: AgentType, - modelName: string, - userProvidedInstructions?: string, - clientOptions?: Record, - ) { - super(type, modelName, userProvidedInstructions); - - // Process client options - this.apiKey = - (clientOptions?.apiKey as string) || process.env.ANTHROPIC_API_KEY || ""; - this.baseURL = (clientOptions?.baseURL as string) || undefined; - - // Get thinking budget if specified - if ( - clientOptions?.thinkingBudget && - typeof clientOptions.thinkingBudget === "number" - ) { - this.thinkingBudget = clientOptions.thinkingBudget; - } - - // Store client options for reference - this.clientOptions = { - apiKey: this.apiKey, - }; - - if (this.baseURL) { - this.clientOptions.baseUrl = this.baseURL; - } - - // Initialize the Anthropic client - this.client = new Anthropic(this.clientOptions); - } - - setViewport(width: number, height: number): void { - this.currentViewport = { width, height }; - } - - setCurrentUrl(url: string): void { - this.currentUrl = url; - } - - setScreenshotProvider(provider: () => Promise): void { - this.screenshotProvider = provider; - } - - setActionHandler(handler: (action: AgentAction) => Promise): void { - this.actionHandler = handler; - } - - /** - * Execute a task with the Anthropic CUA - * This is the main entry point for the agent - * @implements AgentClient.execute - */ - async execute(executionOptions: AgentExecutionOptions): Promise { - const { options, logger } = executionOptions; - const { instruction } = options; - const maxSteps = options.maxSteps || 10; - - let currentStep = 0; - let completed = false; - const actions: AgentAction[] = []; - const messageList: string[] = []; - let finalMessage = ""; - - // Start with the initial instruction - let inputItems: ResponseInputItem[] = - this.createInitialInputItems(instruction); - - logger({ - category: "agent", - message: `Starting Anthropic agent execution with instruction: ${instruction}`, - level: 1, - }); - - try { - // Execute steps until completion or max steps reached - while (!completed && currentStep < maxSteps) { - logger({ - category: "agent", - message: `Executing step ${currentStep + 1}/${maxSteps}`, - level: 2, - }); - - const result = await this.executeStep(inputItems, logger); - - // Add actions to the list - if (result.actions.length > 0) { - logger({ - category: "agent", - message: `Step ${currentStep + 1} performed ${result.actions.length} actions`, - level: 2, - }); - actions.push(...result.actions); - } - - // Update completion status - completed = result.completed; - - // Update the input items for the next step if we're continuing - if (!completed) { - inputItems = result.nextInputItems; - } - - // Record any message for this step - if (result.message) { - messageList.push(result.message); - finalMessage = result.message; - } - - // Increment step counter - currentStep++; - } - - logger({ - category: "agent", - message: `Anthropic agent execution completed: ${completed}, with ${actions.length} total actions performed`, - level: 1, - }); - - // Return the final result - return { - success: completed, - actions, - message: finalMessage, - completed, - }; - } catch (error) { - const errorMessage = - error instanceof Error ? error.message : String(error); - logger({ - category: "agent", - message: `Error executing agent task: ${errorMessage}`, - level: 0, - }); - - return { - success: false, - actions, - message: `Failed to execute task: ${errorMessage}`, - completed: false, - }; - } - } - - async executeStep( - inputItems: ResponseInputItem[], - logger: (message: LogLine) => void, - ): Promise<{ - actions: AgentAction[]; - message: string; - completed: boolean; - nextInputItems: ResponseInputItem[]; - }> { - try { - // Get response from the model - const result = await this.getAction(inputItems); - const content = result.content; - - logger({ - category: "agent", - message: `Received response with ${content.length} content blocks`, - level: 2, - }); - - // Extract actions from the content - const stepActions: AgentAction[] = []; - const toolUseItems: ToolUseItem[] = []; - let message = ""; - - // Process content blocks to find tool use items and text content - for (const block of content) { - // Log the block for debugging - console.log("Processing block:", JSON.stringify(block, null, 2)); - - // Enhanced logging for debugging - logger({ - category: "agent", - message: `Processing block type: ${block.type}, id: ${block.id || "unknown"}`, - level: 2, - }); - - if (block.type === "tool_use") { - // Direct handling of tool_use type - logger({ - category: "agent", - message: `Found tool_use block: ${JSON.stringify(block)}`, - level: 2, - }); - - // Cast to ToolUseItem and add to list - const toolUseItem = block as ToolUseItem; - toolUseItems.push(toolUseItem); - - logger({ - category: "agent", - message: `Added tool_use item: ${toolUseItem.name}, action: ${JSON.stringify(toolUseItem.input)}`, - level: 2, - }); - - // Convert tool use to action and add to actions list - const action = this.convertToolUseToAction(toolUseItem); - if (action) { - logger({ - category: "agent", - message: `Created action from tool_use: ${toolUseItem.name}, action: ${action.type}`, - level: 2, - }); - stepActions.push(action); - } - } else if (block.type === "text") { - // Safe to cast here since we've verified it's a text block - const textBlock = block as unknown as AnthropicTextBlock; - message += textBlock.text + "\n"; - - logger({ - category: "agent", - message: `Found text block: ${textBlock.text.substring(0, 50)}...`, - level: 2, - }); - } else { - logger({ - category: "agent", - message: `Found unknown block type: ${block.type}`, - level: 2, - }); - } - } - - // Execute actions if an action handler is provided - if (this.actionHandler && stepActions.length > 0) { - for (const action of stepActions) { - try { - logger({ - category: "agent", - message: `Executing action: ${action.type}`, - level: 1, - }); - await this.actionHandler(action); - } catch (error) { - const errorMessage = - error instanceof Error ? error.message : String(error); - logger({ - category: "agent", - message: `Error executing action ${action.type}: ${errorMessage}`, - level: 0, - }); - } - } - } - - // Create the assistant response message with all content blocks - const assistantMessage: AnthropicMessage = { - role: "assistant", - content: content as unknown as AnthropicContentBlock[], - }; - - // Keep track of the conversation history by preserving all previous messages - // and adding new messages at the end - const nextInputItems: ResponseInputItem[] = [...inputItems]; - - // Add the assistant message with tool_use blocks to the history - nextInputItems.push(assistantMessage); - - // Generate tool results and add them as a user message - if (toolUseItems.length > 0) { - const toolResults = await this.takeAction(toolUseItems, logger); - - if (toolResults.length > 0) { - // We wrap the tool results in a user message - const userToolResultsMessage: AnthropicMessage = { - role: "user", - content: toolResults as unknown as AnthropicContentBlock[], - }; - nextInputItems.push(userToolResultsMessage); - } - } - - // The step is completed only if there were no tool_use items - const completed = toolUseItems.length === 0; - - logger({ - category: "agent", - message: `Step processed ${toolUseItems.length} tool use items, completed: ${completed}`, - level: 2, - }); - - return { - actions: stepActions, - message: message.trim(), - completed, - nextInputItems, - }; - } catch (error) { - const errorMessage = - error instanceof Error ? error.message : String(error); - logger({ - category: "agent", - message: `Error executing step: ${errorMessage}`, - level: 0, - }); - - throw error; - } - } - - private createInitialInputItems(instruction: string): AnthropicMessage[] { - // For the initial request, we use a simple array with the user's instruction - return [ - { - role: "system", - content: this.userProvidedInstructions, - }, - { - role: "user", - content: instruction, - }, - ]; - } - - async getAction(inputItems: ResponseInputItem[]): Promise<{ - content: AnthropicContentBlock[]; - id: string; - }> { - try { - // For the API request, we use the inputItems directly - // These should already be properly formatted as a sequence of user/assistant messages - const messages: AnthropicMessage[] = []; - - for (const item of inputItems) { - if ("role" in item) { - // Skip system messages as Anthropic requires system as a top-level parameter - if (item.role !== "system") { - messages.push(item); - } - } - // Note: We don't need special handling for tool_result items here anymore - // as they should already be properly wrapped in user messages - } - - // Configure thinking capability if available - const thinking = this.thinkingBudget - ? { type: "enabled" as const, budget_tokens: this.thinkingBudget } - : undefined; - - // Create the request parameters - const requestParams: Record = { - model: this.modelName, - max_tokens: 4096, - messages: messages, - tools: [ - { - type: "computer_20250124", // Use the latest version for Claude 3.7 Sonnet - name: "computer", - display_width_px: this.currentViewport.width, - display_height_px: this.currentViewport.height, - display_number: 1, - }, - ], - betas: ["computer-use-2025-01-24"], - }; - - // Add system parameter if provided - if (this.userProvidedInstructions) { - requestParams.system = this.userProvidedInstructions; - } - - // Add thinking parameter if available - if (thinking) { - requestParams.thinking = thinking; - } - - // Log the request - if (messages.length > 0) { - const firstMessage = messages[0]; - const contentPreview = - typeof firstMessage.content === "string" - ? firstMessage.content.substring(0, 50) - : "complex content"; - - console.log( - `Sending request to Anthropic with ${messages.length} messages and ${messages.length > 0 ? `first message role: ${messages[0].role}, content: ${contentPreview}...` : "no messages"}`, - ); - } - - // Create the message using the Anthropic Messages API - // @ts-expect-error - The Anthropic SDK types are stricter than what we need - const response = await this.client.beta.messages.create(requestParams); - - // Store the message ID for future use - this.lastMessageId = response.id; - - // Return the content and message ID - return { - // Cast the response content to our internal type - content: response.content as unknown as AnthropicContentBlock[], - id: response.id, - }; - } catch (error) { - console.error("Error getting action from Anthropic:", error); - throw error; - } - } - - async takeAction( - toolUseItems: ToolUseItem[], - logger: (message: LogLine) => void, - ): Promise { - const nextInputItems: ResponseInputItem[] = []; - - logger({ - category: "agent", - message: `Taking action on ${toolUseItems.length} tool use items`, - level: 2, - }); - - // Process each tool use item - for (const item of toolUseItems) { - try { - logger({ - category: "agent", - message: `Processing tool use: ${item.name}, id: ${item.id}, action: ${JSON.stringify(item.input)}`, - level: 2, - }); - - // TODO: Normalize and migrate to agentHandler - - // For computer tool, capture screenshot and return image - if (item.name === "computer") { - // Get action type - const action = item.input.action as string; - logger({ - category: "agent", - message: `Computer action type: ${action}`, - level: 2, - }); - - // Capture a screenshot for the response - const screenshot = await this.captureScreenshot(); - logger({ - category: "agent", - message: `Screenshot captured, length: ${screenshot.length}`, - level: 2, - }); - - // Create proper image content block for Anthropic - const imageContent = [ - { - type: "image", - source: { - type: "base64", - media_type: "image/png", - data: screenshot.replace(/^data:image\/png;base64,/, ""), - }, - }, - ]; - - // Add current URL if available - if (this.currentUrl) { - nextInputItems.push({ - type: "tool_result", - tool_use_id: item.id, - content: [ - ...imageContent, - { - type: "text", - text: `Current URL: ${this.currentUrl}`, - }, - ], - }); - } else { - nextInputItems.push({ - type: "tool_result", - tool_use_id: item.id, - content: imageContent, - }); - } - - logger({ - category: "agent", - message: `Added computer tool result for tool_use_id: ${item.id}`, - level: 2, - }); - } else { - // For any other tools, return a simple result as a string - nextInputItems.push({ - type: "tool_result", - tool_use_id: item.id, - content: "Tool executed successfully", - }); - - logger({ - category: "agent", - message: `Added generic tool result for tool ${item.name}, tool_use_id: ${item.id}`, - level: 2, - }); - } - } catch (error) { - const errorMessage = - error instanceof Error ? error.message : String(error); - - logger({ - category: "agent", - message: `Error executing tool use: ${errorMessage}`, - level: 0, - }); - - try { - // For computer tool, try to capture a screenshot even on error - if (item.name === "computer") { - const screenshot = await this.captureScreenshot(); - - nextInputItems.push({ - type: "tool_result", - tool_use_id: item.id, - content: [ - { - type: "image", - source: { - type: "base64", - media_type: "image/png", - data: screenshot.replace(/^data:image\/png;base64,/, ""), - }, - }, - { - type: "text", - text: `Error: ${errorMessage}`, - }, - ], - }); - - logger({ - category: "agent", - message: `Added error tool result with screenshot for tool_use_id: ${item.id}`, - level: 1, - }); - } else { - // For other tools, return an error message as a string - nextInputItems.push({ - type: "tool_result", - tool_use_id: item.id, - content: `Error: ${errorMessage}`, - }); - - logger({ - category: "agent", - message: `Added error tool result for tool_use_id: ${item.id}`, - level: 1, - }); - } - } catch (screenshotError) { - // If we can't capture a screenshot, just send the error - logger({ - category: "agent", - message: `Error capturing screenshot: ${String(screenshotError)}`, - level: 0, - }); - - nextInputItems.push({ - type: "tool_result", - tool_use_id: item.id, - content: `Error: ${errorMessage}`, - }); - - logger({ - category: "agent", - message: `Added text error tool result for tool_use_id: ${item.id}`, - level: 1, - }); - } - } - } - - logger({ - category: "agent", - message: `Prepared ${nextInputItems.length} input items for next request`, - level: 2, - }); - - return nextInputItems; - } - - private convertToolUseToAction(item: ToolUseItem): AgentAction | null { - try { - const { name, input } = item; - - if (name === "computer") { - // For computer actions, format according to the action type - const action = input.action as string; - - if (!action) { - console.warn("Missing action in tool use item:", item); - return null; - } - - // Handle different action types specifically - if (action === "screenshot") { - return { - type: "screenshot", - ...input, - }; - } else if (action === "click") { - return { - type: "click", - x: input.x as number, - y: input.y as number, - button: (input.button as string) || "left", - ...input, - }; - } else if (action === "type") { - return { - type: "type", - text: input.text as string, - ...input, - }; - } else if (action === "keypress") { - return { - type: "keypress", - keys: input.keys as string[], - ...input, - }; - } else if (action === "double_click" || action === "doubleClick") { - return { - type: action, - x: input.x as number, - y: input.y as number, - ...input, - }; - } else if (action === "scroll") { - // Convert Anthropic's coordinate, scroll_amount and scroll_direction into scroll_x and scroll_y - const x = - (input.x as number) || - (input.coordinate ? (input.coordinate as number[])[0] : 0); - const y = - (input.y as number) || - (input.coordinate ? (input.coordinate as number[])[1] : 0); - - // Calculate scroll_x and scroll_y based on scroll_amount and scroll_direction - let scroll_x = 0; - let scroll_y = 0; - - const scrollAmount = (input.scroll_amount as number) || 5; - const scrollMultiplier = 100; // Pixels per unit of scroll_amount - - if (input.scroll_direction) { - const direction = input.scroll_direction as string; - if (direction === "down") { - scroll_y = scrollAmount * scrollMultiplier; - } else if (direction === "up") { - scroll_y = -scrollAmount * scrollMultiplier; - } else if (direction === "right") { - scroll_x = scrollAmount * scrollMultiplier; - } else if (direction === "left") { - scroll_x = -scrollAmount * scrollMultiplier; - } - } else { - // Use direct scroll_x and scroll_y if provided - scroll_x = (input.scroll_x as number) || 0; - scroll_y = (input.scroll_y as number) || 0; - } - - return { - type: "scroll", - x: x, - y: y, - scroll_x: scroll_x, - scroll_y: scroll_y, - ...input, - }; - } else if (action === "move") { - // Handle Anthropic's coordinate format - const coordinates = input.coordinate as number[] | undefined; - const x = coordinates ? coordinates[0] : (input.x as number) || 0; - const y = coordinates ? coordinates[1] : (input.y as number) || 0; - - return { - type: "move", - x: x, - y: y, - ...input, - }; - } else if (action === "drag") { - // Make sure path is properly formatted - const path = - (input.path as { x: number; y: number }[]) || - (input.coordinate - ? [ - { - x: (input.start_coordinate as number[])[0], - y: (input.start_coordinate as number[])[1], - }, - { - x: (input.coordinate as number[])[0], - y: (input.coordinate as number[])[1], - }, - ] - : []); - - return { - type: "drag", - path: path, - ...input, - }; - } else if (action === "wait") { - return { - type: "wait", - ...input, - }; - } else if (action === "key") { - const text = input.text as string; - // Convert common key names to a format our handler can understand - let mappedKey = text; - - if ( - text === "Return" || - text === "return" || - text === "Enter" || - text === "enter" - ) { - mappedKey = "Enter"; - } else if (text === "Tab" || text === "tab") { - mappedKey = "Tab"; - } else if ( - text === "Escape" || - text === "escape" || - text === "Esc" || - text === "esc" - ) { - mappedKey = "Escape"; - } else if (text === "Backspace" || text === "backspace") { - mappedKey = "Backspace"; - } else if ( - text === "Delete" || - text === "delete" || - text === "Del" || - text === "del" - ) { - mappedKey = "Delete"; - } else if (text === "ArrowUp" || text === "Up" || text === "up") { - mappedKey = "ArrowUp"; - } else if ( - text === "ArrowDown" || - text === "Down" || - text === "down" - ) { - mappedKey = "ArrowDown"; - } else if ( - text === "ArrowLeft" || - text === "Left" || - text === "left" - ) { - mappedKey = "ArrowLeft"; - } else if ( - text === "ArrowRight" || - text === "Right" || - text === "right" - ) { - mappedKey = "ArrowRight"; - } - - return { - type: "key", - text: mappedKey, - ...input, - }; - } else if (action === "left_click") { - // Convert left_click to regular click - const coordinates = input.coordinate as number[] | undefined; - const x = coordinates ? coordinates[0] : (input.x as number) || 0; - const y = coordinates ? coordinates[1] : (input.y as number) || 0; - - return { - type: "click", - x: x, - y: y, - button: "left", - ...input, - }; - } else { - // For other computer actions, use the action type directly - console.log(`Using default action mapping for ${action}`); - return { - type: action, - ...input, - }; - } - } else if (name === "str_replace_editor" || name === "bash") { - // For editor or bash tools - return { - type: name, - params: input, - }; - } - - console.warn(`Unknown tool name: ${name}`); - return null; - } catch (error) { - console.error("Error converting tool use to action:", error); - return null; - } - } - - async captureScreenshot(options?: { - base64Image?: string; - currentUrl?: string; - }): Promise { - // Use provided options if available - if (options?.base64Image) { - return `data:image/png;base64,${options.base64Image}`; - } - - // Use the screenshot provider if available - if (this.screenshotProvider) { - try { - const base64Image = await this.screenshotProvider(); - return `data:image/png;base64,${base64Image}`; - } catch (error) { - console.error("Error capturing screenshot:", error); - throw error; - } - } - - throw new AgentScreenshotProviderError( - "`screenshotProvider` has not been set. " + - "Please call `setScreenshotProvider()` with a valid function that returns a base64-encoded image", - ); - } -} diff --git a/lib/agent/OpenAICUAClient.ts b/lib/agent/OpenAICUAClient.ts deleted file mode 100644 index 6a494300b..000000000 --- a/lib/agent/OpenAICUAClient.ts +++ /dev/null @@ -1,582 +0,0 @@ -import OpenAI from "openai"; -import { LogLine } from "../../types/log"; -import { - AgentAction, - AgentResult, - AgentType, - AgentExecutionOptions, - ResponseInputItem, - ResponseItem, - ComputerCallItem, - FunctionCallItem, -} from "@/types/agent"; -import { AgentClient } from "./AgentClient"; -import { AgentScreenshotProviderError } from "@/types/stagehandErrors"; - -/** - * Client for OpenAI's Computer Use Assistant API - * This implementation uses the official OpenAI Responses API for Computer Use - */ -export class OpenAICUAClient extends AgentClient { - private apiKey: string; - private organization?: string; - private baseURL: string; - private client: OpenAI; - public lastResponseId?: string; - private currentViewport = { width: 1024, height: 768 }; - private currentUrl?: string; - private screenshotProvider?: () => Promise; - private actionHandler?: (action: AgentAction) => Promise; - private reasoningItems: Map = new Map(); - private environment: string = "browser"; // "browser", "mac", "windows", or "ubuntu" - - constructor( - type: AgentType, - modelName: string, - userProvidedInstructions?: string, - clientOptions?: Record, - ) { - super(type, modelName, userProvidedInstructions); - - // Process client options - this.apiKey = - (clientOptions?.apiKey as string) || process.env.OPENAI_API_KEY || ""; - this.organization = - (clientOptions?.organization as string) || process.env.OPENAI_ORG; - - // Get environment if specified - if ( - clientOptions?.environment && - typeof clientOptions.environment === "string" - ) { - this.environment = clientOptions.environment; - } - - // Store client options for reference - this.clientOptions = { - apiKey: this.apiKey, - }; - - // Initialize the OpenAI client - this.client = new OpenAI(this.clientOptions); - } - - setViewport(width: number, height: number): void { - this.currentViewport = { width, height }; - } - - setCurrentUrl(url: string): void { - this.currentUrl = url; - } - - setScreenshotProvider(provider: () => Promise): void { - this.screenshotProvider = provider; - } - - setActionHandler(handler: (action: AgentAction) => Promise): void { - this.actionHandler = handler; - } - - /** - * Execute a task with the OpenAI CUA - * This is the main entry point for the agent - * @implements AgentClient.execute - */ - async execute(executionOptions: AgentExecutionOptions): Promise { - const { options, logger } = executionOptions; - const { instruction } = options; - const maxSteps = options.maxSteps || 10; - - let currentStep = 0; - let completed = false; - const actions: AgentAction[] = []; - const messageList: string[] = []; - let finalMessage = ""; - this.reasoningItems.clear(); // Clear any previous reasoning items - - // Start with the initial instruction - let inputItems = this.createInitialInputItems(instruction); - let previousResponseId: string | undefined = undefined; - - try { - // Execute steps until completion or max steps reached - while (!completed && currentStep < maxSteps) { - logger({ - category: "agent", - message: `Executing step ${currentStep + 1}/${maxSteps}`, - level: 2, - }); - - const result = await this.executeStep( - inputItems, - previousResponseId, - logger, - ); - - // Add actions to the list - actions.push(...result.actions); - - // Update completion status - completed = result.completed; - - // Store the previous response ID for the next request - previousResponseId = result.responseId; - - // Update the input items for the next step if we're continuing - if (!completed) { - inputItems = result.nextInputItems; - } - - // Record any message for this step - if (result.message) { - messageList.push(result.message); - finalMessage = result.message; - } - - // Increment step counter - currentStep++; - } - - // Return the final result - return { - success: completed, - actions, - message: finalMessage, - completed, - }; - } catch (error) { - const errorMessage = - error instanceof Error ? error.message : String(error); - logger({ - category: "agent", - message: `Error executing agent task: ${errorMessage}`, - level: 0, - }); - - return { - success: false, - actions, - message: `Failed to execute task: ${errorMessage}`, - completed: false, - }; - } - } - - /** - * Execute a single step of the agent - * This coordinates the flow: Request → Get Action → Execute Action - */ - async executeStep( - inputItems: ResponseInputItem[], - previousResponseId: string | undefined, - logger: (message: LogLine) => void, - ): Promise<{ - actions: AgentAction[]; - message: string; - completed: boolean; - nextInputItems: ResponseInputItem[]; - responseId: string; - }> { - try { - // Get response from the model - const result = await this.getAction(inputItems, previousResponseId); - const output = result.output; - const responseId = result.responseId; - - // Add any reasoning items to our map - for (const item of output) { - if (item.type === "reasoning") { - this.reasoningItems.set(item.id, item); - } - } - - // Extract actions from the output - const stepActions: AgentAction[] = []; - for (const item of output) { - if (item.type === "computer_call" && this.isComputerCallItem(item)) { - const action = this.convertComputerCallToAction(item); - if (action) { - stepActions.push(action); - } - } else if ( - item.type === "function_call" && - this.isFunctionCallItem(item) - ) { - const action = this.convertFunctionCallToAction(item); - if (action) { - stepActions.push(action); - } - } - } - - // Extract message text - let message = ""; - for (const item of output) { - if (item.type === "message") { - if (item.content && Array.isArray(item.content)) { - for (const content of item.content) { - if (content.type === "output_text" && content.text) { - message += content.text + "\n"; - } - } - } - } - } - - // Take actions and get results - const nextInputItems = await this.takeAction(output, logger); - - // Check if completed - const completed = - output.length === 0 || - output.every( - (item) => item.type === "message" || item.type === "reasoning", - ); - - return { - actions: stepActions, - message: message.trim(), - completed, - nextInputItems, - responseId, - }; - } catch (error) { - const errorMessage = - error instanceof Error ? error.message : String(error); - logger({ - category: "agent", - message: `Error executing step: ${errorMessage}`, - level: 0, - }); - - throw error; - } - } - - private isComputerCallItem(item: ResponseItem): item is ComputerCallItem { - return ( - item.type === "computer_call" && - "call_id" in item && - "action" in item && - typeof item.action === "object" - ); - } - - private isFunctionCallItem(item: ResponseItem): item is FunctionCallItem { - return ( - item.type === "function_call" && - "call_id" in item && - "name" in item && - "arguments" in item - ); - } - - private createInitialInputItems(instruction: string): ResponseInputItem[] { - // For the initial request, we use a simple array with the user's instruction - return [ - { - role: "system", - content: this.userProvidedInstructions, - }, - { - role: "user", - content: instruction, - }, - ]; - } - - async getAction( - inputItems: ResponseInputItem[], - previousResponseId?: string, - ): Promise<{ - output: ResponseItem[]; - responseId: string; - }> { - try { - // Create the request parameters - const requestParams: Record = { - model: this.modelName, - tools: [ - { - type: "computer_use_preview", - display_width: this.currentViewport.width, - display_height: this.currentViewport.height, - environment: this.environment, - }, - ], - input: inputItems, - truncation: "auto", - }; - - // Add previous_response_id if available - if (previousResponseId) { - requestParams.previous_response_id = previousResponseId; - } - - // Create the response using the OpenAI Responses API - // @ts-expect-error - Force type to match what the OpenAI SDK expects - const response = await this.client.responses.create(requestParams); - - // Store the response ID for future use - this.lastResponseId = response.id; - - // Return the output and response ID - return { - output: response.output as unknown as ResponseItem[], - responseId: response.id, - }; - } catch (error) { - console.error("Error getting action from OpenAI:", error); - throw error; - } - } - - async takeAction( - output: ResponseItem[], - logger: (message: LogLine) => void, - ): Promise { - const nextInputItems: ResponseInputItem[] = []; - - // Add any computer calls to process - for (const item of output) { - if (item.type === "computer_call" && this.isComputerCallItem(item)) { - // Execute the action - try { - const action = this.convertComputerCallToAction(item); - - if (action && this.actionHandler) { - await this.actionHandler(action); - } - - // Capture a screenshot - const screenshot = await this.captureScreenshot(); - - // Create a computer_call_output for the next request - const outputItem = { - type: "computer_call_output" as const, - call_id: item.call_id, - output: { - type: "input_image" as const, - image_url: screenshot, - }, - } as ResponseInputItem; - - // Add current URL if available - if (this.currentUrl) { - const computerCallOutput = outputItem as { - type: "computer_call_output"; - call_id: string; - output: { - type: "input_image"; - image_url: string; - current_url?: string; - }; - acknowledged_safety_checks?: Array<{ - id: string; - code: string; - message: string; - }>; - }; - computerCallOutput.output.current_url = this.currentUrl; - } - - // Add any safety checks that need to be acknowledged - if ( - item.pending_safety_checks && - item.pending_safety_checks.length > 0 - ) { - const computerCallOutput = outputItem as { - type: "computer_call_output"; - call_id: string; - output: { - type: "input_image"; - image_url: string; - }; - acknowledged_safety_checks?: Array<{ - id: string; - code: string; - message: string; - }>; - }; - computerCallOutput.acknowledged_safety_checks = - item.pending_safety_checks; - } - - nextInputItems.push(outputItem); - } catch (error) { - const errorMessage = - error instanceof Error ? error.message : String(error); - - logger({ - category: "agent", - message: `Error executing computer call: ${errorMessage}`, - level: 0, - }); - - try { - // Capture a screenshot even on error - const screenshot = await this.captureScreenshot(); - - const errorOutputItem = { - type: "computer_call_output" as const, - call_id: item.call_id, - output: { - type: "input_image" as const, - image_url: screenshot, - error: errorMessage, - }, - } as ResponseInputItem; - - // Add current URL if available - if (this.currentUrl) { - const computerCallOutput = errorOutputItem as { - type: "computer_call_output"; - call_id: string; - output: { - type: "input_image"; - image_url: string; - current_url?: string; - }; - acknowledged_safety_checks?: Array<{ - id: string; - code: string; - message: string; - }>; - }; - computerCallOutput.output.current_url = this.currentUrl; - } - - // Add any safety checks that need to be acknowledged - if ( - item.pending_safety_checks && - item.pending_safety_checks.length > 0 - ) { - const computerCallOutput = errorOutputItem as { - type: "computer_call_output"; - call_id: string; - output: { - type: "input_image"; - image_url: string; - }; - acknowledged_safety_checks?: Array<{ - id: string; - code: string; - message: string; - }>; - }; - computerCallOutput.acknowledged_safety_checks = - item.pending_safety_checks; - } - - nextInputItems.push(errorOutputItem); - } catch (screenshotError) { - // If we can't capture a screenshot, just send the error - logger({ - category: "agent", - message: `Error capturing screenshot: ${String(screenshotError)}`, - level: 0, - }); - - // For error cases without a screenshot, we need to use a string output - nextInputItems.push({ - type: "computer_call_output", - call_id: item.call_id, - output: `Error: ${errorMessage}`, - } as ResponseInputItem); - } - } - } else if ( - item.type === "function_call" && - this.isFunctionCallItem(item) - ) { - // Execute the function - try { - const action = this.convertFunctionCallToAction(item); - - if (action && this.actionHandler) { - await this.actionHandler(action); - } - - // Add the result - nextInputItems.push({ - type: "function_call_output", - call_id: item.call_id, - output: "success", - }); - } catch (error) { - const errorMessage = - error instanceof Error ? error.message : String(error); - - logger({ - category: "agent", - message: `Error executing function call: ${errorMessage}`, - level: 0, - }); - - nextInputItems.push({ - type: "function_call_output", - call_id: item.call_id, - output: `Error: ${errorMessage}`, - }); - } - } - } - - return nextInputItems; - } - - private convertComputerCallToAction( - call: ComputerCallItem, - ): AgentAction | null { - const { action } = call; - - // Instead of wrapping the action in a params object, spread the action properties directly - // This ensures properties like x, y, button, etc. are directly accessible on the AgentAction - return { - type: action.type as string, - ...action, // Spread all properties from the action - }; - } - - private convertFunctionCallToAction( - call: FunctionCallItem, - ): AgentAction | null { - try { - const args = JSON.parse(call.arguments); - - return { - type: call.name, - params: args, - }; - } catch (error) { - console.error("Error parsing function call arguments:", error); - return null; - } - } - - async captureScreenshot(options?: { - base64Image?: string; - currentUrl?: string; - }): Promise { - // Use provided options if available - if (options?.base64Image) { - return `data:image/png;base64,${options.base64Image}`; - } - - // Use the screenshot provider if available - if (this.screenshotProvider) { - try { - const base64Image = await this.screenshotProvider(); - return `data:image/png;base64,${base64Image}`; - } catch (error) { - console.error("Error capturing screenshot:", error); - throw error; - } - } - - throw new AgentScreenshotProviderError( - "`screenshotProvider` has not been set. " + - "Please call `setScreenshotProvider()` with a valid function that returns a base64-encoded image", - ); - } -} diff --git a/lib/agent/StagehandAgent.ts b/lib/agent/StagehandAgent.ts deleted file mode 100644 index ab65d1efb..000000000 --- a/lib/agent/StagehandAgent.ts +++ /dev/null @@ -1,52 +0,0 @@ -import { LogLine } from "@/types/log"; -import { - AgentExecuteOptions, - AgentResult, - AgentExecutionOptions, -} from "@/types/agent"; -import { AgentClient } from "./AgentClient"; - -/** - * Main interface for agent operations in Stagehand - * This class provides methods for executing tasks with an agent - */ -export class StagehandAgent { - private client: AgentClient; - private logger: (message: LogLine) => void; - - constructor(client: AgentClient, logger: (message: LogLine) => void) { - this.client = client; - this.logger = logger; - } - - async execute( - optionsOrInstruction: AgentExecuteOptions | string, - ): Promise { - const options = - typeof optionsOrInstruction === "string" - ? { instruction: optionsOrInstruction } - : optionsOrInstruction; - - this.logger({ - category: "agent", - message: `Executing agent task: ${options.instruction}`, - level: 1, - }); - - const executionOptions: AgentExecutionOptions = { - options, - logger: this.logger, - retries: 3, - }; - - return await this.client.execute(executionOptions); - } - - getModelName(): string { - return this.client.modelName; - } - - getAgentType(): string { - return this.client.type; - } -} diff --git a/lib/api.ts b/lib/api.ts deleted file mode 100644 index e05b96aad..000000000 --- a/lib/api.ts +++ /dev/null @@ -1,247 +0,0 @@ -import { z } from "zod"; -import zodToJsonSchema from "zod-to-json-schema"; -import { - ApiResponse, - ExecuteActionParams, - StagehandAPIConstructorParams, - StartSessionParams, - StartSessionResult, -} from "../types/api"; -import { LogLine } from "../types/log"; -import { GotoOptions } from "../types/playwright"; -import { - ActOptions, - ActResult, - AgentConfig, - ExtractOptions, - ExtractResult, - ObserveOptions, - ObserveResult, -} from "../types/stagehand"; -import { AgentExecuteOptions, AgentResult } from "."; -import { - StagehandAPIUnauthorizedError, - StagehandHttpError, - StagehandAPIError, - StagehandServerError, - StagehandResponseBodyError, - StagehandResponseParseError, -} from "../types/stagehandApiErrors"; - -export class StagehandAPI { - private apiKey: string; - private projectId: string; - private sessionId?: string; - private modelApiKey: string; - private logger: (message: LogLine) => void; - - constructor({ apiKey, projectId, logger }: StagehandAPIConstructorParams) { - this.apiKey = apiKey; - this.projectId = projectId; - this.logger = logger; - } - - async init({ - modelName, - modelApiKey, - domSettleTimeoutMs, - verbose, - debugDom, - systemPrompt, - selfHeal, - waitForCaptchaSolves, - actionTimeoutMs, - browserbaseSessionCreateParams, - browserbaseSessionID, - }: StartSessionParams): Promise { - if (!modelApiKey) { - throw new StagehandAPIError("modelApiKey is required"); - } - this.modelApiKey = modelApiKey; - const sessionResponse = await this.request("/sessions/start", { - method: "POST", - body: JSON.stringify({ - modelName, - domSettleTimeoutMs, - verbose, - debugDom, - systemPrompt, - selfHeal, - waitForCaptchaSolves, - actionTimeoutMs, - browserbaseSessionCreateParams, - browserbaseSessionID, - }), - }); - - if (sessionResponse.status === 401) { - throw new StagehandAPIUnauthorizedError( - "Unauthorized. Ensure you provided a valid API key and that it is whitelisted.", - ); - } else if (sessionResponse.status !== 200) { - console.log(await sessionResponse.text()); - throw new StagehandHttpError(`Unknown error: ${sessionResponse.status}`); - } - - const sessionResponseBody = - (await sessionResponse.json()) as ApiResponse; - - if (sessionResponseBody.success === false) { - throw new StagehandAPIError(sessionResponseBody.message); - } - - this.sessionId = sessionResponseBody.data.sessionId; - - return sessionResponseBody.data; - } - - async act(options: ActOptions | ObserveResult): Promise { - return this.execute({ - method: "act", - args: { ...options }, - }); - } - - async extract( - options: ExtractOptions, - ): Promise> { - if (!options.schema) { - return this.execute>({ - method: "extract", - args: {}, - }); - } - const parsedSchema = zodToJsonSchema(options.schema); - return this.execute>({ - method: "extract", - args: { ...options, schemaDefinition: parsedSchema }, - }); - } - - async observe(options?: ObserveOptions): Promise { - return this.execute({ - method: "observe", - args: { ...options }, - }); - } - - async goto(url: string, options?: GotoOptions): Promise { - return this.execute({ - method: "navigate", - args: { url, options }, - }); - } - - async agentExecute( - agentConfig: AgentConfig, - executeOptions: AgentExecuteOptions, - ): Promise { - return this.execute({ - method: "agentExecute", - args: { agentConfig, executeOptions }, - }); - } - - async end(): Promise { - const url = `/sessions/${this.sessionId}/end`; - return await this.request(url, { - method: "POST", - }); - } - - private async execute({ - method, - args, - params, - }: ExecuteActionParams): Promise { - const urlParams = new URLSearchParams(params as Record); - const queryString = urlParams.toString(); - const url = `/sessions/${this.sessionId}/${method}${queryString ? `?${queryString}` : ""}`; - - const response = await this.request(url, { - method: "POST", - body: JSON.stringify(args), - }); - - if (!response.ok) { - const errorBody = await response.text(); - throw new StagehandHttpError( - `HTTP error! status: ${response.status}, body: ${errorBody}`, - ); - } - - if (!response.body) { - throw new StagehandResponseBodyError(); - } - - const reader = response.body.getReader(); - const decoder = new TextDecoder(); - let buffer = ""; - - while (true) { - const { value, done } = await reader.read(); - - if (done && !buffer) { - return null; - } - - buffer += decoder.decode(value, { stream: true }); - const lines = buffer.split("\n\n"); - buffer = lines.pop() || ""; - - for (const line of lines) { - if (!line.startsWith("data: ")) continue; - - try { - const eventData = JSON.parse(line.slice(6)); - - if (eventData.type === "system") { - if (eventData.data.status === "error") { - throw new StagehandServerError(eventData.data.error); - } - if (eventData.data.status === "finished") { - return eventData.data.result as T; - } - } else if (eventData.type === "log") { - this.logger(eventData.data.message); - } - } catch (e) { - console.error("Error parsing event data:", e); - throw new StagehandResponseParseError( - "Failed to parse server response", - ); - } - } - - if (done) break; - } - } - - private async request( - path: string, - options: RequestInit = {}, - ): Promise { - const defaultHeaders: Record = { - "x-bb-api-key": this.apiKey, - "x-bb-project-id": this.projectId, - "x-bb-session-id": this.sessionId, - // we want real-time logs, so we stream the response - "x-stream-response": "true", - "x-model-api-key": this.modelApiKey, - }; - - if (options.method === "POST" && options.body) { - defaultHeaders["Content-Type"] = "application/json"; - } - - const response = await fetch(`${process.env.STAGEHAND_API_URL}${path}`, { - ...options, - headers: { - ...defaultHeaders, - ...options.headers, - }, - }); - - return response; - } -} diff --git a/lib/cache.ts b/lib/cache.ts deleted file mode 100644 index a9e2a981d..000000000 --- a/lib/cache.ts +++ /dev/null @@ -1,98 +0,0 @@ -import fs from "fs"; -const observationsPath = "./.cache/observations.json"; -const actionsPath = "./.cache/actions.json"; - -/** - * A file system cache to skip inference when repeating steps - * It also acts as the source of truth for identifying previously seen actions and observations - */ -class Cache { - disabled: boolean; - - constructor({ disabled = false } = {}) { - this.disabled = disabled; - if (!this.disabled) { - this.initCache(); - } - } - - readObservations() { - if (this.disabled) { - return {}; - } - try { - return JSON.parse(fs.readFileSync(observationsPath, "utf8")); - } catch (error) { - console.error("Error reading from observations.json", error); - return {}; - } - } - - readActions() { - if (this.disabled) { - return {}; - } - try { - return JSON.parse(fs.readFileSync(actionsPath, "utf8")); - } catch (error) { - console.error("Error reading from actions.json", error); - return {}; - } - } - - writeObservations({ - key, - value, - }: { - key: string; - value: { id: string; result: string }; - }) { - if (this.disabled) { - return; - } - - const observations = this.readObservations(); - observations[key] = value; - fs.writeFileSync(observationsPath, JSON.stringify(observations, null, 2)); - } - - writeActions({ - key, - value, - }: { - key: string; - value: { id: string; result: string }; - }) { - if (this.disabled) { - return; - } - - const actions = this.readActions(); - actions[key] = value; - fs.writeFileSync(actionsPath, JSON.stringify(actions, null, 2)); - } - - evictCache() { - throw new Error("implement me"); - } - - private initCache() { - if (this.disabled) { - return; - } - const cacheDir = ".cache"; - - if (!fs.existsSync(cacheDir)) { - fs.mkdirSync(cacheDir); - } - if (!fs.existsSync(actionsPath)) { - fs.writeFileSync(actionsPath, JSON.stringify({})); - } - - if (!fs.existsSync(observationsPath)) { - fs.writeFileSync(observationsPath, JSON.stringify({})); - } - } -} - -export default Cache; diff --git a/lib/cache/ActionCache.ts b/lib/cache/ActionCache.ts deleted file mode 100644 index b54801d6c..000000000 --- a/lib/cache/ActionCache.ts +++ /dev/null @@ -1,158 +0,0 @@ -import { LogLine } from "../../types/log"; -import { BaseCache, CacheEntry } from "./BaseCache"; - -export interface PlaywrightCommand { - method: string; - args: string[]; -} - -export interface ActionEntry extends CacheEntry { - data: { - playwrightCommand: PlaywrightCommand; - componentString: string; - xpaths: string[]; - newStepString: string; - completed: boolean; - previousSelectors: string[]; - action: string; - }; -} - -/** - * ActionCache handles logging and retrieving actions along with their Playwright commands. - */ -export class ActionCache extends BaseCache { - constructor( - logger: (message: LogLine) => void, - cacheDir?: string, - cacheFile?: string, - ) { - super(logger, cacheDir, cacheFile || "action_cache.json"); - } - - public async addActionStep({ - url, - action, - previousSelectors, - playwrightCommand, - componentString, - xpaths, - newStepString, - completed, - requestId, - }: { - url: string; - action: string; - previousSelectors: string[]; - playwrightCommand: PlaywrightCommand; - componentString: string; - requestId: string; - xpaths: string[]; - newStepString: string; - completed: boolean; - }): Promise { - this.logger({ - category: "action_cache", - message: "adding action step to cache", - level: 1, - auxiliary: { - action: { - value: action, - type: "string", - }, - requestId: { - value: requestId, - type: "string", - }, - url: { - value: url, - type: "string", - }, - previousSelectors: { - value: JSON.stringify(previousSelectors), - type: "object", - }, - }, - }); - - await this.set( - { url, action, previousSelectors }, - { - playwrightCommand, - componentString, - xpaths, - newStepString, - completed, - previousSelectors, - action, - }, - requestId, - ); - } - - /** - * Retrieves all actions for a specific trajectory. - * @param trajectoryId - Unique identifier for the trajectory. - * @param requestId - The identifier for the current request. - * @returns An array of TrajectoryEntry objects or null if not found. - */ - public async getActionStep({ - url, - action, - previousSelectors, - requestId, - }: { - url: string; - action: string; - previousSelectors: string[]; - requestId: string; - }): Promise { - const data = await super.get({ url, action, previousSelectors }, requestId); - if (!data) { - return null; - } - - return data; - } - - public async removeActionStep(cacheHashObj: { - url: string; - action: string; - previousSelectors: string[]; - requestId: string; - }): Promise { - await super.delete(cacheHashObj); - } - - /** - * Clears all actions for a specific trajectory. - * @param trajectoryId - Unique identifier for the trajectory. - * @param requestId - The identifier for the current request. - */ - public async clearAction(requestId: string): Promise { - await super.deleteCacheForRequestId(requestId); - this.logger({ - category: "action_cache", - message: "cleared action for ID", - level: 1, - auxiliary: { - requestId: { - value: requestId, - type: "string", - }, - }, - }); - } - - /** - * Resets the entire action cache. - */ - public async resetCache(): Promise { - await super.resetCache(); - this.logger({ - category: "action_cache", - message: "Action cache has been reset.", - level: 1, - }); - } -} diff --git a/lib/cache/BaseCache.ts b/lib/cache/BaseCache.ts deleted file mode 100644 index 18ef61690..000000000 --- a/lib/cache/BaseCache.ts +++ /dev/null @@ -1,568 +0,0 @@ -import * as fs from "fs"; -import * as path from "path"; -import * as crypto from "crypto"; -import { LogLine } from "../../types/log"; - -export interface CacheEntry { - timestamp: number; - data: unknown; - requestId: string; -} - -export interface CacheStore { - [key: string]: CacheEntry; -} - -export class BaseCache { - private readonly CACHE_MAX_AGE_MS = 7 * 24 * 60 * 60 * 1000; // 1 week in milliseconds - private readonly CLEANUP_PROBABILITY = 0.01; // 1% chance - - protected cacheDir: string; - protected cacheFile: string; - protected lockFile: string; - protected logger: (message: LogLine) => void; - - private readonly LOCK_TIMEOUT_MS = 1_000; - protected lockAcquired = false; - protected lockAcquireFailures = 0; - - // Added for request ID tracking - protected requestIdToUsedHashes: { [key: string]: string[] } = {}; - - constructor( - logger: (message: LogLine) => void, - cacheDir: string = path.join(process.cwd(), "tmp", ".cache"), - cacheFile: string = "cache.json", - ) { - this.logger = logger; - this.cacheDir = cacheDir; - this.cacheFile = path.join(cacheDir, cacheFile); - this.lockFile = path.join(cacheDir, "cache.lock"); - this.ensureCacheDirectory(); - this.setupProcessHandlers(); - } - - private setupProcessHandlers(): void { - const releaseLockAndExit = () => { - this.releaseLock(); - process.exit(); - }; - - process.on("exit", releaseLockAndExit); - process.on("SIGINT", releaseLockAndExit); - process.on("SIGTERM", releaseLockAndExit); - process.on("uncaughtException", (err) => { - this.logger({ - category: "base_cache", - message: "uncaught exception", - level: 2, - auxiliary: { - error: { - value: err.message, - type: "string", - }, - trace: { - value: err.stack, - type: "string", - }, - }, - }); - if (this.lockAcquired) { - releaseLockAndExit(); - } - }); - } - - protected ensureCacheDirectory(): void { - if (!fs.existsSync(this.cacheDir)) { - fs.mkdirSync(this.cacheDir, { recursive: true }); - this.logger({ - category: "base_cache", - message: "created cache directory", - level: 1, - auxiliary: { - cacheDir: { - value: this.cacheDir, - type: "string", - }, - }, - }); - } - } - - protected createHash(data: unknown): string { - const hash = crypto.createHash("sha256"); - return hash.update(JSON.stringify(data)).digest("hex"); - } - - protected sleep(ms: number): Promise { - return new Promise((resolve) => setTimeout(resolve, ms)); - } - - public async acquireLock(): Promise { - const startTime = Date.now(); - while (Date.now() - startTime < this.LOCK_TIMEOUT_MS) { - try { - if (fs.existsSync(this.lockFile)) { - const lockAge = Date.now() - fs.statSync(this.lockFile).mtimeMs; - if (lockAge > this.LOCK_TIMEOUT_MS) { - fs.unlinkSync(this.lockFile); - this.logger({ - category: "base_cache", - message: "Stale lock file removed", - level: 1, - }); - } - } - - fs.writeFileSync(this.lockFile, process.pid.toString(), { flag: "wx" }); - this.lockAcquireFailures = 0; - this.lockAcquired = true; - this.logger({ - category: "base_cache", - message: "Lock acquired", - level: 1, - }); - return true; - } catch (e) { - this.logger({ - category: "base_cache", - message: "error acquiring lock", - level: 2, - auxiliary: { - trace: { - value: e.stack, - type: "string", - }, - message: { - value: e.message, - type: "string", - }, - }, - }); - await this.sleep(5); - } - } - this.logger({ - category: "base_cache", - message: "Failed to acquire lock after timeout", - level: 2, - }); - this.lockAcquireFailures++; - if (this.lockAcquireFailures >= 3) { - this.logger({ - category: "base_cache", - message: - "Failed to acquire lock 3 times in a row. Releasing lock manually.", - level: 1, - }); - this.releaseLock(); - } - return false; - } - - public releaseLock(): void { - try { - if (fs.existsSync(this.lockFile)) { - fs.unlinkSync(this.lockFile); - this.logger({ - category: "base_cache", - message: "Lock released", - level: 1, - }); - } - this.lockAcquired = false; - } catch (error) { - this.logger({ - category: "base_cache", - message: "error releasing lock", - level: 2, - auxiliary: { - error: { - value: error.message, - type: "string", - }, - trace: { - value: error.stack, - type: "string", - }, - }, - }); - } - } - - /** - * Cleans up stale cache entries that exceed the maximum age. - */ - public async cleanupStaleEntries(): Promise { - if (!(await this.acquireLock())) { - this.logger({ - category: "llm_cache", - message: "failed to acquire lock for cleanup", - level: 2, - }); - return; - } - - try { - const cache = this.readCache(); - const now = Date.now(); - let entriesRemoved = 0; - - for (const [hash, entry] of Object.entries(cache)) { - if (now - entry.timestamp > this.CACHE_MAX_AGE_MS) { - delete cache[hash]; - entriesRemoved++; - } - } - - if (entriesRemoved > 0) { - this.writeCache(cache); - this.logger({ - category: "llm_cache", - message: "cleaned up stale cache entries", - level: 1, - auxiliary: { - entriesRemoved: { - value: entriesRemoved.toString(), - type: "integer", - }, - }, - }); - } - } catch (error) { - this.logger({ - category: "llm_cache", - message: "error during cache cleanup", - level: 2, - auxiliary: { - error: { - value: error.message, - type: "string", - }, - trace: { - value: error.stack, - type: "string", - }, - }, - }); - } finally { - this.releaseLock(); - } - } - - protected readCache(): CacheStore { - if (fs.existsSync(this.cacheFile)) { - try { - const data = fs.readFileSync(this.cacheFile, "utf-8"); - return JSON.parse(data) as CacheStore; - } catch (error) { - this.logger({ - category: "base_cache", - message: "error reading cache file. resetting cache.", - level: 1, - auxiliary: { - error: { - value: error.message, - type: "string", - }, - trace: { - value: error.stack, - type: "string", - }, - }, - }); - this.resetCache(); - return {}; - } - } - return {}; - } - - protected writeCache(cache: CacheStore): void { - try { - fs.writeFileSync(this.cacheFile, JSON.stringify(cache, null, 2)); - this.logger({ - category: "base_cache", - message: "Cache written to file", - level: 1, - }); - } catch (error) { - this.logger({ - category: "base_cache", - message: "error writing cache file", - level: 2, - auxiliary: { - error: { - value: error.message, - type: "string", - }, - trace: { - value: error.stack, - type: "string", - }, - }, - }); - } finally { - this.releaseLock(); - } - } - - /** - * Retrieves data from the cache based on the provided options. - * @param hashObj - The options used to generate the cache key. - * @param requestId - The identifier for the current request. - * @returns The cached data if available, otherwise null. - */ - public async get( - hashObj: Record | string, - requestId: string, - ): Promise { - if (!(await this.acquireLock())) { - this.logger({ - category: "base_cache", - message: "Failed to acquire lock for getting cache", - level: 2, - }); - return null; - } - - try { - const hash = this.createHash(hashObj); - const cache = this.readCache(); - - if (cache[hash]) { - this.trackRequestIdUsage(requestId, hash); - return cache[hash].data; - } - return null; - } catch (error) { - this.logger({ - category: "base_cache", - message: "error getting cache. resetting cache.", - level: 1, - auxiliary: { - error: { - value: error.message, - type: "string", - }, - trace: { - value: error.stack, - type: "string", - }, - }, - }); - - this.resetCache(); - return null; - } finally { - this.releaseLock(); - } - } - - /** - * Stores data in the cache based on the provided options and requestId. - * @param hashObj - The options used to generate the cache key. - * @param data - The data to be cached. - * @param requestId - The identifier for the cache entry. - */ - public async set( - hashObj: Record, - data: T["data"], - requestId: string, - ): Promise { - if (!(await this.acquireLock())) { - this.logger({ - category: "base_cache", - message: "Failed to acquire lock for setting cache", - level: 2, - }); - return; - } - - try { - const hash = this.createHash(hashObj); - const cache = this.readCache(); - cache[hash] = { - data, - timestamp: Date.now(), - requestId, - }; - - this.writeCache(cache); - this.trackRequestIdUsage(requestId, hash); - } catch (error) { - this.logger({ - category: "base_cache", - message: "error setting cache. resetting cache.", - level: 1, - auxiliary: { - error: { - value: error.message, - type: "string", - }, - trace: { - value: error.stack, - type: "string", - }, - }, - }); - - this.resetCache(); - } finally { - this.releaseLock(); - - if (Math.random() < this.CLEANUP_PROBABILITY) { - this.cleanupStaleEntries(); - } - } - } - - public async delete(hashObj: Record): Promise { - if (!(await this.acquireLock())) { - this.logger({ - category: "base_cache", - message: "Failed to acquire lock for removing cache entry", - level: 2, - }); - return; - } - - try { - const hash = this.createHash(hashObj); - const cache = this.readCache(); - - if (cache[hash]) { - delete cache[hash]; - this.writeCache(cache); - } else { - this.logger({ - category: "base_cache", - message: "Cache entry not found to delete", - level: 1, - }); - } - } catch (error) { - this.logger({ - category: "base_cache", - message: "error removing cache entry", - level: 2, - auxiliary: { - error: { - value: error.message, - type: "string", - }, - trace: { - value: error.stack, - type: "string", - }, - }, - }); - } finally { - this.releaseLock(); - } - } - - /** - * Tracks the usage of a hash with a specific requestId. - * @param requestId - The identifier for the current request. - * @param hash - The cache key hash. - */ - protected trackRequestIdUsage(requestId: string, hash: string): void { - this.requestIdToUsedHashes[requestId] ??= []; - this.requestIdToUsedHashes[requestId].push(hash); - } - - /** - * Deletes all cache entries associated with a specific requestId. - * @param requestId - The identifier for the request whose cache entries should be deleted. - */ - public async deleteCacheForRequestId(requestId: string): Promise { - if (!(await this.acquireLock())) { - this.logger({ - category: "base_cache", - message: "Failed to acquire lock for deleting cache", - level: 2, - }); - return; - } - try { - const cache = this.readCache(); - const hashes = this.requestIdToUsedHashes[requestId] ?? []; - let entriesRemoved = 0; - for (const hash of hashes) { - if (cache[hash]) { - delete cache[hash]; - entriesRemoved++; - } - } - if (entriesRemoved > 0) { - this.writeCache(cache); - } else { - this.logger({ - category: "base_cache", - message: "no cache entries found for requestId", - level: 1, - auxiliary: { - requestId: { - value: requestId, - type: "string", - }, - }, - }); - } - // Remove the requestId from the mapping after deletion - delete this.requestIdToUsedHashes[requestId]; - } catch (error) { - this.logger({ - category: "base_cache", - message: "error deleting cache for requestId", - level: 2, - auxiliary: { - error: { - value: error.message, - type: "string", - }, - trace: { - value: error.stack, - type: "string", - }, - requestId: { - value: requestId, - type: "string", - }, - }, - }); - } finally { - this.releaseLock(); - } - } - - /** - * Resets the entire cache by clearing the cache file. - */ - public resetCache(): void { - try { - fs.writeFileSync(this.cacheFile, "{}"); - this.requestIdToUsedHashes = {}; // Reset requestId tracking - } catch (error) { - this.logger({ - category: "base_cache", - message: "error resetting cache", - level: 2, - auxiliary: { - error: { - value: error.message, - type: "string", - }, - trace: { - value: error.stack, - type: "string", - }, - }, - }); - } finally { - this.releaseLock(); - } - } -} diff --git a/lib/cache/LLMCache.ts b/lib/cache/LLMCache.ts deleted file mode 100644 index 53bd78213..000000000 --- a/lib/cache/LLMCache.ts +++ /dev/null @@ -1,48 +0,0 @@ -import { BaseCache, CacheEntry } from "./BaseCache"; - -export class LLMCache extends BaseCache { - constructor( - logger: (message: { - category?: string; - message: string; - level?: number; - }) => void, - cacheDir?: string, - cacheFile?: string, - ) { - super(logger, cacheDir, cacheFile || "llm_calls.json"); - } - - /** - * Overrides the get method to track used hashes by requestId. - * @param options - The options used to generate the cache key. - * @param requestId - The identifier for the current request. - * @returns The cached data if available, otherwise null. - */ - public async get( - options: Record, - requestId: string, - ): Promise { - const data = await super.get(options, requestId); - return data as T | null; // TODO: remove this cast - } - - /** - * Overrides the set method to include cache cleanup logic. - * @param options - The options used to generate the cache key. - * @param data - The data to be cached. - * @param requestId - The identifier for the current request. - */ - public async set( - options: Record, - data: unknown, - requestId: string, - ): Promise { - await super.set(options, data, requestId); - this.logger({ - category: "llm_cache", - message: "Cache miss - saved new response", - level: 1, - }); - } -} diff --git a/lib/dom/DomChunk.ts b/lib/dom/DomChunk.ts deleted file mode 100644 index 0b2232efa..000000000 --- a/lib/dom/DomChunk.ts +++ /dev/null @@ -1,6 +0,0 @@ -export interface DomChunk { - startOffset: number; - endOffset: number; - outputString: string; - selectorMap: Record; -} diff --git a/lib/dom/ElementContainer.ts b/lib/dom/ElementContainer.ts deleted file mode 100644 index c831e0489..000000000 --- a/lib/dom/ElementContainer.ts +++ /dev/null @@ -1,98 +0,0 @@ -import { StagehandContainer } from "./StagehandContainer"; - -/** - * The ElementContainer class is a container implementation for a specific - * HTML element. - * - * Unlike `GlobalPageContainer`, which manages the entire page, - * this class focuses on one particular `HTMLElement`. Operations - * such as `scrollTo` and `scrollIntoView` apply to that element - * rather than `window`. - */ -export class ElementContainer extends StagehandContainer { - /** - * Creates an instance of `ElementContainer` tied to a specific element. - * @param el - The scrollable `HTMLElement` that this container controls. - */ - constructor(private el: HTMLElement) { - super(); - } - - public getRootElement(): HTMLElement { - return this.el; - } - - /** - * Retrieves the height of the visible viewport within this element - * (`el.clientHeight`). - * - * @returns The visible (client) height of the element, in pixels. - */ - public getViewportHeight(): number { - return this.el.clientHeight; - } - - public getScrollHeight(): number { - return this.el.scrollHeight; - } - - /** - * Returns the element's current vertical scroll offset. - */ - public getScrollPosition(): number { - return this.el.scrollTop; - } - - /** - * Smoothly scrolls this element to the specified vertical offset, and - * waits for the scrolling to complete. - * - * @param offset - The scroll offset (in pixels) from the top of the element. - * @returns A promise that resolves once scrolling is finished. - */ - public async scrollTo(offset: number): Promise { - await new Promise((resolve) => setTimeout(resolve, 1500)); - this.el.scrollTo({ top: offset, behavior: "smooth" }); - await this.waitForScrollEnd(); - } - - /** - * Scrolls this element so that the given `element` is visible, or - * scrolls to the top if none is provided. Smoothly animates the scroll - * and waits until it finishes. - * - * @param element - The child element to bring into view. If omitted, scrolls to top. - * @returns A promise that resolves once scrolling completes. - */ - public async scrollIntoView(element?: HTMLElement): Promise { - if (!element) { - this.el.scrollTo({ top: 0, behavior: "smooth" }); - } else { - element.scrollIntoView(); - } - await this.waitForScrollEnd(); - } - - /** - * Internal helper that waits until scrolling in this element has - * fully stopped. It listens for scroll events on the element, - * resetting a short timer every time a scroll occurs, and resolves - * once there's no scroll for ~100ms. - * - * @returns A promise that resolves when scrolling has finished. - */ - private async waitForScrollEnd(): Promise { - return new Promise((resolve) => { - let scrollEndTimer: number; - const handleScroll = () => { - clearTimeout(scrollEndTimer); - scrollEndTimer = window.setTimeout(() => { - this.el.removeEventListener("scroll", handleScroll); - resolve(); - }, 100); - }; - this.el.addEventListener("scroll", handleScroll, { passive: true }); - handleScroll(); - }); - } -} diff --git a/lib/dom/GlobalPageContainer.ts b/lib/dom/GlobalPageContainer.ts deleted file mode 100644 index 46270fd1d..000000000 --- a/lib/dom/GlobalPageContainer.ts +++ /dev/null @@ -1,91 +0,0 @@ -import { StagehandContainer } from "./StagehandContainer"; -import { calculateViewportHeight } from "./utils"; - -/** - * The `GlobalPageContainer` class is a container implementation for the entire - * webpage or global document. - * - * This container manages the global `window` scroll position and provides - * measurements using `document.documentElement`. It extends `StagehandContainer` - * to unify scrolling/height logic with other container types, such as element - * based containers from the `ElementContainer` class. - */ -export class GlobalPageContainer extends StagehandContainer { - public getRootElement(): HTMLElement { - return document.body; - } - - /** - * Calculates the viewport height for the entire page, using a helper. - * The helper returns 75% of the window height, to ensure that we don't - * miss any content that may be behind sticky elements like nav bars. - * - * @returns The current height of the global viewport, in pixels. - */ - public getViewportHeight(): number { - return calculateViewportHeight(); - } - - public getScrollHeight(): number { - return document.documentElement.scrollHeight; - } - - public getScrollPosition(): number { - return window.scrollY; - } - - /** - * Smoothly scrolls the page to the specified vertical offset, and then - * waits until scrolling has stopped. There is a delay built in to allow - * for lazy loading and other asynchronous content to load. - * - * @param offset - The desired scroll offset from the top of the page. - * @returns A promise that resolves once scrolling is complete. - */ - public async scrollTo(offset: number): Promise { - await new Promise((resolve) => setTimeout(resolve, 1500)); - window.scrollTo({ top: offset, behavior: "smooth" }); - await this.waitForScrollEnd(); - } - - /** - * Scrolls the page so that a given element is visible, or scrolls to the top - * if no element is specified. Uses smooth scrolling and waits for it to complete. - * - * @param element - The DOM element to bring into view. If omitted, scrolls to top. - * @returns A promise that resolves once scrolling is complete. - */ - public async scrollIntoView(element?: HTMLElement): Promise { - if (!element) { - window.scrollTo({ top: 0, behavior: "smooth" }); - } else { - const rect = element.getBoundingClientRect(); - const currentY = window.scrollY || document.documentElement.scrollTop; - const elementY = currentY + rect.top - window.innerHeight * 0.25; - window.scrollTo({ top: elementY, behavior: "smooth" }); - } - await this.waitForScrollEnd(); - } - - /** - * Internal helper that waits until the global scroll activity has stopped. - * It listens for scroll events, resetting a short timer every time a scroll - * occurs, and resolves once there's no scroll for ~100ms. - * - * @returns A promise that resolves when scrolling has finished. - */ - private async waitForScrollEnd(): Promise { - return new Promise((resolve) => { - let scrollEndTimer: number; - const handleScroll = () => { - clearTimeout(scrollEndTimer); - scrollEndTimer = window.setTimeout(() => { - window.removeEventListener("scroll", handleScroll); - resolve(); - }, 100); - }; - window.addEventListener("scroll", handleScroll, { passive: true }); - handleScroll(); - }); - } -} diff --git a/lib/dom/StagehandContainer.ts b/lib/dom/StagehandContainer.ts deleted file mode 100644 index 94e107e93..000000000 --- a/lib/dom/StagehandContainer.ts +++ /dev/null @@ -1,128 +0,0 @@ -import { DomChunk } from "@/lib/dom/DomChunk"; -import { collectCandidateElements } from "@/lib/dom/candidateCollector"; - -/** - * The `StagehandContainer` class defines an abstract interface for - * scrolling and DOM inspection across various container types e.g., - * the entire page (GlobalPageContainer), a specific sub element of the DOM, - * (ElementContainer). - */ -export abstract class StagehandContainer { - public abstract getViewportHeight(): number; - public abstract getScrollHeight(): number; - public abstract scrollTo(offset: number): Promise; - public abstract getRootElement(): HTMLElement | Document; - public abstract scrollIntoView(element?: HTMLElement): Promise; - public abstract getScrollPosition(): number; - - /** - * Collects multiple "DOM chunks" by scrolling through the container - * in increments from `startOffset` to `endOffset`. At each scroll - * position, the function extracts a snapshot of "candidate elements" - * using `collectCandidateElements`. - * - * Each chunk represents a subset of the DOM at a particular - * vertical scroll offset, including: - * - * - `startOffset` & `endOffset`: The vertical scroll bounds for this chunk. - * - `outputString`: A serialized representation of extracted DOM text. - * - `selectorMap`: A mapping of temporary indices to the actual element(s) - * that were collected in this chunk, useful for further processing. - * - * @param startOffset - The initial scroll offset from which to begin collecting. - * @param endOffset - The maximum scroll offset to collect up to. - * @param chunkSize - The vertical increment to move between each chunk. - * @param scrollTo - Whether we should scroll to the chunk - * @param scrollBackToTop - Whether to scroll the container back to the top once finished. - * @param candidateContainer - Optionally, a specific container element within - * the root for which to collect data. If omitted, uses `this.getRootElement()`. - * - * @returns A promise that resolves with an array of `DomChunk` objects. - * - * ### How It Works - * - * 1. **Scroll Range Calculation**: - * - Computes `maxOffset` as the maximum offset that can be scrolled - * (`scrollHeight - viewportHeight`). - * - Restricts `endOffset` to not exceed `maxOffset`. - * - * 2. **Chunk Iteration**: - * - Loops from `startOffset` to `endOffset` in steps of `chunkSize`. - * - For each offset `current`, we call `this.scrollTo(current)` - * to position the container. - * - * 3. **Element Collection**: - * - Invokes `collectCandidateElements` on either `candidateContainer` - * (if provided) or the result of `this.getRootElement()`. - * - This returns both an `outputString` (serialized text) - * and a `selectorMap` of found elements for that section of the DOM. - * - * 4. **Chunk Assembly**: - * - Creates a `DomChunk` object for the current offset range, - * storing `outputString`, `selectorMap`, and scroll offsets. - * - Pushes it onto the `chunks` array. - * - * 5. **Scroll Reset**: - * - Once iteration completes, if `scrollBackToTop` is `true`, - * we scroll back to offset `0`. - */ - public async collectDomChunks( - startOffset: number, - endOffset: number, - chunkSize: number, - scrollTo: boolean = true, - scrollBackToTop: boolean = true, - candidateContainer?: HTMLElement, - ): Promise { - const chunks: DomChunk[] = []; - let maxOffset = this.getScrollHeight(); - let current = startOffset; - let finalEnd = endOffset; - - let index = 0; - - while (current <= finalEnd) { - // Move the container's scroll position - if (scrollTo) { - await this.scrollTo(current); - } - - // Collect the candidate elements at this offset - const rootCandidate = - candidateContainer || (this.getRootElement() as HTMLElement); - const { outputString, selectorMap } = await collectCandidateElements( - rootCandidate, - index, - ); - - chunks.push({ - startOffset: current, - endOffset: current + chunkSize, - outputString, - selectorMap, - }); - - index += Object.keys(selectorMap).length; - current += chunkSize; - - // Only extend finalEnd if there is no candidateContainer - // (meaning we're looking at the entire scrollable area) - if (!candidateContainer && current > endOffset) { - // Check if new content extended the scroll height - const newScrollHeight = this.getScrollHeight(); - if (newScrollHeight > maxOffset) { - maxOffset = newScrollHeight; - } - if (newScrollHeight > finalEnd) { - finalEnd = newScrollHeight; - } - } - } - - if (scrollBackToTop) { - await this.scrollTo(0); - } - - return chunks; - } -} diff --git a/lib/dom/candidateCollector.ts b/lib/dom/candidateCollector.ts deleted file mode 100644 index e577653f1..000000000 --- a/lib/dom/candidateCollector.ts +++ /dev/null @@ -1,139 +0,0 @@ -import { - isElementNode, - isTextNode, - isInteractiveElement, - isLeafElement, - isVisible, - isTextVisible, - isActive, -} from "./elementCheckUtils"; -import { generateXPathsForElement as generateXPaths } from "./xpathUtils"; - -const xpathCache: Map = new Map(); - -/** - * `collectCandidateElements` performs a depth-first traversal (despite the BFS naming) of the given `candidateContainerRoot` - * to find “candidate elements” or text nodes that meet certain criteria (e.g., visible, active, - * interactive, or leaf). This function does not scroll the page; it only collects nodes from - * the in-memory DOM structure starting at `candidateContainerRoot`. - * - * @param candidateContainerRoot - The HTMLElement to search within - * @param indexOffset - A numeric offset used to label/number your candidate elements - * @returns { outputString, selectorMap } - */ -export async function collectCandidateElements( - candidateContainerRoot: HTMLElement, - indexOffset: number = 0, -): Promise<{ - outputString: string; - selectorMap: Record; -}> { - const DOMQueue: ChildNode[] = [...candidateContainerRoot.childNodes]; - const candidateElements: ChildNode[] = []; - - while (DOMQueue.length > 0) { - const node = DOMQueue.pop(); - let shouldAdd = false; - - if (node && isElementNode(node)) { - for (let i = node.childNodes.length - 1; i >= 0; i--) { - DOMQueue.push(node.childNodes[i]); - } - - if (isInteractiveElement(node)) { - if (isActive(node) && isVisible(node)) { - shouldAdd = true; - } - } - if (isLeafElement(node)) { - if (isActive(node) && isVisible(node)) { - shouldAdd = true; - } - } - } - - if (node && isTextNode(node) && isTextVisible(node)) { - shouldAdd = true; - } - - if (shouldAdd) { - candidateElements.push(node); - } - } - - const selectorMap: Record = {}; - let outputString = ""; - - const xpathLists = await Promise.all( - candidateElements.map((elem) => { - if (xpathCache.has(elem)) { - return Promise.resolve(xpathCache.get(elem)!); - } - return generateXPaths(elem).then((xpaths: string[]) => { - xpathCache.set(elem, xpaths); - return xpaths; - }); - }), - ); - - candidateElements.forEach((elem, idx) => { - const xpaths = xpathLists[idx]; - let elemOutput = ""; - - if (isTextNode(elem)) { - const textContent = elem.textContent?.trim(); - if (textContent) { - elemOutput += `${idx + indexOffset}:${textContent}\n`; - } - } else if (isElementNode(elem)) { - const tagName = elem.tagName.toLowerCase(); - const attributes = collectEssentialAttributes(elem); - const opening = `<${tagName}${attributes ? " " + attributes : ""}>`; - const closing = ``; - const textContent = elem.textContent?.trim() || ""; - elemOutput += `${idx + indexOffset}:${opening}${textContent}${closing}\n`; - } - - outputString += elemOutput; - selectorMap[idx + indexOffset] = xpaths; - }); - - return { outputString, selectorMap }; -} - -/** - * Collects essential attributes from an element. - * @param element The DOM element. - * @returns A string of formatted attributes. - */ -function collectEssentialAttributes(element: Element): string { - const essentialAttributes = [ - "id", - "class", - "href", - "src", - "aria-label", - "aria-name", - "aria-role", - "aria-description", - "aria-expanded", - "aria-haspopup", - "type", - "value", - ]; - - const attrs: string[] = essentialAttributes - .map((attr) => { - const value = element.getAttribute(attr); - return value ? `${attr}="${value}"` : ""; - }) - .filter((attr) => attr !== ""); - - Array.from(element.attributes).forEach((attr) => { - if (attr.name.startsWith("data-")) { - attrs.push(`${attr.name}="${attr.value}"`); - } - }); - - return attrs.join(" "); -} diff --git a/lib/dom/containerFactory.ts b/lib/dom/containerFactory.ts deleted file mode 100644 index 2375a5737..000000000 --- a/lib/dom/containerFactory.ts +++ /dev/null @@ -1,16 +0,0 @@ -import { StagehandContainer } from "./StagehandContainer"; -import { GlobalPageContainer } from "./GlobalPageContainer"; -import { ElementContainer } from "./ElementContainer"; - -/** - * Decide which container to create. - */ -export function createStagehandContainer( - obj: Window | HTMLElement, -): StagehandContainer { - if (obj instanceof Window) { - return new GlobalPageContainer(); - } else { - return new ElementContainer(obj); - } -} diff --git a/lib/dom/elementCheckUtils.ts b/lib/dom/elementCheckUtils.ts deleted file mode 100644 index b62637730..000000000 --- a/lib/dom/elementCheckUtils.ts +++ /dev/null @@ -1,168 +0,0 @@ -export function isElementNode(node: Node): node is Element { - return node.nodeType === Node.ELEMENT_NODE; -} - -export function isTextNode(node: Node): node is Text { - return node.nodeType === Node.TEXT_NODE && Boolean(node.textContent?.trim()); -} - -const leafElementDenyList = ["SVG", "IFRAME", "SCRIPT", "STYLE", "LINK"]; - -const interactiveElementTypes = [ - "A", - "BUTTON", - "DETAILS", - "EMBED", - "INPUT", - "LABEL", - "MENU", - "MENUITEM", - "OBJECT", - "SELECT", - "TEXTAREA", - "SUMMARY", -]; - -const interactiveRoles = [ - "button", - "menu", - "menuitem", - "link", - "checkbox", - "radio", - "slider", - "tab", - "tabpanel", - "textbox", - "combobox", - "grid", - "listbox", - "option", - "progressbar", - "scrollbar", - "searchbox", - "switch", - "tree", - "treeitem", - "spinbutton", - "tooltip", -]; -const interactiveAriaRoles = ["menu", "menuitem", "button"]; - -/* - * Checks if an element is visible and therefore relevant for LLMs to consider. We check: - * - Size - * - Display properties - * - Opacity - * If the element is a child of a previously hidden element, it should not be included, so we don't consider downstream effects of a parent element here - */ -export const isVisible = (element: Element) => { - const rect = element.getBoundingClientRect(); - // Ensure the element is within the viewport - if ( - rect.width === 0 || - rect.height === 0 || - rect.top < 0 || - rect.top > window.innerHeight - ) { - return false; - } - if (!isTopElement(element, rect)) { - return false; - } - - const visible = element.checkVisibility({ - checkOpacity: true, - checkVisibilityCSS: true, - }); - - return visible; -}; - -export const isTextVisible = (element: ChildNode) => { - const range = document.createRange(); - range.selectNodeContents(element); - const rect = range.getBoundingClientRect(); - - if ( - rect.width === 0 || - rect.height === 0 || - rect.top < 0 || - rect.top > window.innerHeight - ) { - return false; - } - const parent = element.parentElement; - if (!parent) { - return false; - } - - const visible = parent.checkVisibility({ - checkOpacity: true, - checkVisibilityCSS: true, - }); - - return visible; -}; - -export function isTopElement(elem: ChildNode, rect: DOMRect) { - const points = [ - { x: rect.left + rect.width * 0.25, y: rect.top + rect.height * 0.25 }, - { x: rect.left + rect.width * 0.75, y: rect.top + rect.height * 0.25 }, - { x: rect.left + rect.width * 0.25, y: rect.top + rect.height * 0.75 }, - { x: rect.left + rect.width * 0.75, y: rect.top + rect.height * 0.75 }, - { x: rect.left + rect.width / 2, y: rect.top + rect.height / 2 }, - ]; - - return points.some((point) => { - const topEl = document.elementFromPoint(point.x, point.y); - let current = topEl; - while (current && current !== document.body) { - if (current.isSameNode(elem)) { - return true; - } - current = current.parentElement; - } - return false; - }); -} - -export const isActive = (element: Element) => { - if ( - element.hasAttribute("disabled") || - element.hasAttribute("hidden") || - element.getAttribute("aria-disabled") === "true" - ) { - return false; - } - - return true; -}; -export const isInteractiveElement = (element: Element) => { - const elementType = element.tagName; - const elementRole = element.getAttribute("role"); - const elementAriaRole = element.getAttribute("aria-role"); - - return ( - (elementType && interactiveElementTypes.includes(elementType)) || - (elementRole && interactiveRoles.includes(elementRole)) || - (elementAriaRole && interactiveAriaRoles.includes(elementAriaRole)) - ); -}; - -export const isLeafElement = (element: Element) => { - if (element.textContent === "") { - return false; - } - - if (element.childNodes.length === 0) { - return !leafElementDenyList.includes(element.tagName); - } - - // This case ensures that extra context will be included for simple element nodes that contain only text - if (element.childNodes.length === 1 && isTextNode(element.childNodes[0])) { - return true; - } - - return false; -}; diff --git a/lib/dom/genDomScripts.ts b/lib/dom/genDomScripts.ts deleted file mode 100644 index d3c3edc1c..000000000 --- a/lib/dom/genDomScripts.ts +++ /dev/null @@ -1,29 +0,0 @@ -/** - * We have a collection of typescript functions that we need to run in the browser. - * First, we build them into a single js file - * Second, due to framework differences we need to get our script content as a string to avoid pathing issues due to file routing in frameworks like Next.js - * Playwright allows us to pass in script content directly as a string instead of reading a file from a path - * https://github.com/browserbase/stagehand/issues/180 - * - * We can't rely on the normal build process for stagehand, because we need our script content as a string so that the import *just works* - */ -import fs from "fs"; -import path from "path"; -import esbuild from "esbuild"; - -fs.mkdirSync(path.join(__dirname, "./build"), { recursive: true }); - -esbuild.buildSync({ - entryPoints: [path.join(__dirname, "index.ts")], - bundle: true, - outdir: path.join(__dirname, "build"), -}); - -const scriptContent = fs.readFileSync( - path.join(__dirname, "./build/index.js"), - "utf8", -); - -const output = `export const scriptContent = ${JSON.stringify(scriptContent)};`; - -fs.writeFileSync(path.join(__dirname, "./build/scriptContent.ts"), output); diff --git a/lib/dom/global.d.ts b/lib/dom/global.d.ts deleted file mode 100644 index 742c15433..000000000 --- a/lib/dom/global.d.ts +++ /dev/null @@ -1,37 +0,0 @@ -import { StagehandContainer } from "./StagehandContainer"; - -export {}; -declare global { - interface Window { - chunkNumber: number; - showChunks?: boolean; - processDom: (chunksSeen: Array) => Promise<{ - outputString: string; - selectorMap: Record; - chunk: number; - chunks: number[]; - }>; - processAllOfDom: (xpath?: string) => Promise<{ - outputString: string; - selectorMap: Record; - }>; - createStagehandContainer: (obj: Window | HTMLElement) => StagehandContainer; - waitForDomSettle: () => Promise; - __playwright?: unknown; - __pw_manual?: unknown; - __PW_inspect?: unknown; - storeDOM: (xpath?: string) => string; - restoreDOM: (storedDOM: string, xpath?: string) => void; - createTextBoundingBoxes: (xpath?: string) => void; - getElementBoundingBoxes: (xpath: string) => Array<{ - text: string; - top: number; - left: number; - width: number; - height: number; - }>; - getScrollableElementXpaths: (topN?: number) => Promise; - getNodeFromXpath: (xpath: string) => Node | null; - waitForElementScrollEnd: (element: HTMLElement) => Promise; - } -} diff --git a/lib/dom/index.ts b/lib/dom/index.ts deleted file mode 100644 index 96b53216a..000000000 --- a/lib/dom/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from "./process"; -export * from "./utils"; diff --git a/lib/dom/process.ts b/lib/dom/process.ts deleted file mode 100644 index c9aa28408..000000000 --- a/lib/dom/process.ts +++ /dev/null @@ -1,567 +0,0 @@ -import { generateXPathsForElement as generateXPaths } from "./xpathUtils"; -import { - calculateViewportHeight, - canElementScroll, - getNodeFromXpath, - waitForDomSettle, - waitForElementScrollEnd, -} from "./utils"; -import { createStagehandContainer } from "./containerFactory"; -import { StagehandContainer } from "./StagehandContainer"; -import { GlobalPageContainer } from "@/lib/dom/GlobalPageContainer"; -import { ElementContainer } from "@/lib/dom/ElementContainer"; -import { DomChunk } from "@/lib/dom/DomChunk"; -import { StagehandDomProcessError } from "@/types/stagehandErrors"; - -/** - * Finds and returns a list of scrollable elements on the page, - * ordered from the element with the largest scrollHeight to the smallest. - * - * @param topN Optional maximum number of scrollable elements to return. - * If not provided, all found scrollable elements are returned. - * @returns An array of HTMLElements sorted by descending scrollHeight. - */ -export function getScrollableElements(topN?: number): HTMLElement[] { - // Get the root element - const docEl = document.documentElement; - - // 1) Initialize an array to hold all scrollable elements. - // Always include the root element as a fallback. - const scrollableElements: HTMLElement[] = [docEl]; - - // 2) Scan all elements to find potential scrollable containers. - // A candidate must have a scrollable overflow style and extra scrollable content. - const allElements = document.querySelectorAll("*"); - for (const elem of allElements) { - const style = window.getComputedStyle(elem); - const overflowY = style.overflowY; - - const isPotentiallyScrollable = - overflowY === "auto" || overflowY === "scroll" || overflowY === "overlay"; - - if (isPotentiallyScrollable) { - const candidateScrollDiff = elem.scrollHeight - elem.clientHeight; - // Only consider this element if it actually has extra scrollable content - // and it can truly scroll. - if (candidateScrollDiff > 0 && canElementScroll(elem)) { - scrollableElements.push(elem); - } - } - } - - // 3) Sort the scrollable elements from largest scrollHeight to smallest. - scrollableElements.sort((a, b) => b.scrollHeight - a.scrollHeight); - - // 4) If a topN limit is specified, return only the first topN elements. - if (topN !== undefined) { - return scrollableElements.slice(0, topN); - } - - // Return all found scrollable elements if no limit is provided. - return scrollableElements; -} - -/** - * Calls getScrollableElements, then for each element calls generateXPaths, - * and returns the first XPath for each. - * - * @param topN (optional) integer limit on how many scrollable elements to process - * @returns string[] list of XPaths (1 for each scrollable element) - */ -export async function getScrollableElementXpaths( - topN?: number, -): Promise { - const scrollableElems = getScrollableElements(topN); - const xpaths = []; - for (const elem of scrollableElems) { - const allXPaths = await generateXPaths(elem); - const firstXPath = allXPaths?.[0] || ""; - xpaths.push(firstXPath); - } - return xpaths; -} - -export function getNearestScrollableParent(el: HTMLElement): HTMLElement { - // 1) Get *all* scrollable elements on the page - // (You could pass a large topN or omit it for “all”) - const allScrollables = getScrollableElements(); - - // 2) Climb up the DOM tree - let current: HTMLElement | null = el; - while (current) { - // If `current` is in the scrollable list, we have our nearest scrollable parent - if (allScrollables.includes(current)) { - return current; - } - current = current.parentElement; - } - - // 3) If we exhaust the ancestors, default to root - return document.documentElement; -} - -/** - * processDom - * ---------- - * This function now just picks a single chunk index, - * creates a container, and calls collectDomChunk to get a single DomChunk. - * - * @param chunksSeen - The chunks we've seen so far - */ -export async function processDom(chunksSeen: number[]) { - // 1) choose a chunk index from chunksSeen - const { chunk, chunksArray } = await pickChunk(chunksSeen); - - // 2) create a container for the entire page - const container = new GlobalPageContainer(); - - // 3) pick a chunk size, e.g. container.getViewportHeight() - const chunkSize = container.getViewportHeight(); - - // 4) compute startOffset from `chunkIndex * chunkSize` - const startOffset = chunk * chunkSize; - // if we only want a single chunk, pass the same value for `endOffset` - const endOffset = startOffset; - - // 5) collectAllDomChunks with identical start & end => exactly 1 iteration - const domChunks = await container.collectDomChunks( - startOffset, - endOffset, - chunkSize, - true, - false, // scrollBackToTop - container.getRootElement(), // BFS entire doc - ); - - // We expect exactly 1 chunk - const [domChunk] = domChunks; - if (!domChunk) { - // fallback, no chunk - return { - outputString: "", - selectorMap: {}, - chunk, - chunks: chunksArray, - }; - } - - console.log("Extracted DOM chunk:\n", domChunk.outputString); - - return { - outputString: domChunk.outputString, - selectorMap: domChunk.selectorMap, - chunk, - chunks: chunksArray, - }; -} - -/** - * processAllOfDom - * --------------- - * If an xpath is provided, we find that element and build an appropriate container - * (global or element) based on the nearest scrollable parent. Then we chunk from - * startOffset..endOffset. We BFS within the element's subtree (candidateContainer). - */ -export async function processAllOfDom(xpath?: string) { - let candidateElementContainer: HTMLElement | null = null; - let scrollTarget: StagehandContainer; - - if (xpath) { - // 1) Find the element - const node = getNodeFromXpath(xpath) as HTMLElement | null; - - if (node) { - candidateElementContainer = node; - console.log(`Found element via XPath: ${xpath}`); - - // 2) Always find the nearest scrollable parent - const scrollableElem = getNearestScrollableParent( - candidateElementContainer, - ); - if (scrollableElem === document.documentElement) { - scrollTarget = new GlobalPageContainer(); - } else { - scrollTarget = new ElementContainer(scrollableElem); - } - - // 3) Scroll the candidateElementContainer into view - // (use the container's scroll logic) - await scrollTarget.scrollIntoView(candidateElementContainer); - - // 4) Measure the container’s new scroll offset AFTER we've scrolled - const startOffset = scrollTarget.getScrollPosition(); - - // Now check if the element “fits” in the container’s viewport - const scrollTargetHeight = scrollTarget.getViewportHeight(); - const candidateElementContainerHeight = - candidateElementContainer.scrollHeight; - - if (candidateElementContainerHeight <= scrollTargetHeight) { - // Single-chunk approach - console.log( - "Element is smaller/equal to container’s viewport. Doing single chunk.", - ); - - // We'll define a "single-chunk" scenario by telling - // the container to gather from startOffset..startOffset - // so it only processes exactly one chunk iteration - const domChunks = await scrollTarget.collectDomChunks( - startOffset, // startOffset - startOffset, // endOffset => same as start => 1 chunk - 1, // chunkSize=1 => doesn't matter, because start==end means exactly 1 iteration - true, - true, - candidateElementContainer, - ); - - const singleChunkOutput = combineChunks(domChunks); - console.log( - "Final output (single-chunk):", - singleChunkOutput.outputString, - ); - return singleChunkOutput; - } - - console.log("Element is bigger. Doing multi-chunk approach."); - } else { - console.warn(`XPath not found: ${xpath}. Using entire doc.`); - } - } else { - const scrollableElems = getScrollableElements(1); - const mainScrollable = scrollableElems[0]; - - scrollTarget = - mainScrollable === document.documentElement - ? createStagehandContainer(window) - : createStagehandContainer(mainScrollable); - } - - const startOffset = scrollTarget.getScrollPosition(); - const viewportHeight = scrollTarget.getViewportHeight(); - const maxScroll = candidateElementContainer - ? startOffset + candidateElementContainer.scrollHeight - : scrollTarget.getScrollHeight(); - const chunkSize = viewportHeight; - - console.log("processAllOfDom chunk-based from", startOffset, "to", maxScroll); - - const domChunks = await scrollTarget.collectDomChunks( - startOffset, - maxScroll, - chunkSize, - true, - true, - candidateElementContainer ?? undefined, - ); - - const finalOutput = combineChunks(domChunks); - console.log( - "All DOM elements combined (chunk-based):", - finalOutput.outputString, - ); - return finalOutput; -} - -function combineChunks(domChunks: DomChunk[]): { - outputString: string; - selectorMap: Record; -} { - const outputString = domChunks.map((c: DomChunk) => c.outputString).join(""); - let finalSelectorMap: Record = {}; - domChunks.forEach((c: DomChunk) => { - finalSelectorMap = { ...finalSelectorMap, ...c.selectorMap }; - }); - return { outputString, selectorMap: finalSelectorMap }; -} - -/** - * Stores either the entire DOM (if no `xpath` is given), - * or the specific element that `xpath` points to. - * Returns the outer HTML of what is stored. - */ -export function storeDOM(xpath?: string): string { - if (!xpath) { - const originalDOM = document.body.cloneNode(true) as HTMLElement; - console.log("DOM state stored (root)."); - return originalDOM.outerHTML; - } else { - const node = getNodeFromXpath(xpath) as HTMLElement | null; - - if (!node) { - console.error( - `storeDOM: No element found for xpath: ${xpath}. Returning empty string.`, - ); - return ""; - } - - console.log(`DOM state stored (element at xpath: ${xpath}).`); - return node.outerHTML; - } -} - -/** - * Restores either the entire DOM (if no `xpath` is given), - * or the specific element that `xpath` points to, based on `storedDOM`. - */ -export function restoreDOM(storedDOM: string, xpath?: string): void { - console.log("Restoring DOM..."); - - if (!storedDOM) { - console.error("No DOM state was provided."); - return; - } - - if (!xpath) { - document.body.innerHTML = storedDOM; - console.log("DOM restored (root)."); - } else { - const node = getNodeFromXpath(xpath) as HTMLElement | null; - - if (!node) { - console.error( - `restoreDOM: No element found for xpath: ${xpath}. Cannot restore.`, - ); - return; - } - - node.outerHTML = storedDOM; - console.log(`DOM restored (element at xpath: ${xpath}).`); - } -} - -export function createTextBoundingBoxes(xpath?: string): void { - const style = document.createElement("style"); - document.head.appendChild(style); - if (style.sheet) { - style.sheet.insertRule( - ` - .stagehand-highlighted-word, .stagehand-space { - border: 0px solid orange; - display: inline-block !important; - visibility: visible; - } - `, - 0, - ); - - style.sheet.insertRule( - ` - code .stagehand-highlighted-word, code .stagehand-space, - pre .stagehand-highlighted-word, pre .stagehand-space { - white-space: pre-wrap; - display: inline !important; - } - `, - 1, - ); - } - - /** - * Applies highlighting to every text node under `root`. - * If `root` is the entire `document`, we query "body *". - * If `root` is a specific `HTMLElement`, we query "*". - */ - function applyHighlighting(root: Document | HTMLElement): void { - // If root is a Document, highlight everything under . Otherwise, highlight all children of that HTMLElement. - const containerSelector = root instanceof Document ? "body *" : "*"; - - root.querySelectorAll(containerSelector).forEach((element) => { - if ( - element.closest && - element.closest(".stagehand-nav, .stagehand-marker") - ) { - return; - } - if (["SCRIPT", "STYLE", "IFRAME", "INPUT"].includes(element.tagName)) { - return; - } - - const childNodes = Array.from(element.childNodes); - childNodes.forEach((node) => { - if (node.nodeType === 3 && node.textContent?.trim().length > 0) { - const textContent = node.textContent.replace(/\u00A0/g, " "); - const tokens = textContent.split(/(\s+)/g); // Split text by spaces - const fragment = document.createDocumentFragment(); - const parentIsCode = element.tagName === "CODE"; - - tokens.forEach((token) => { - const span = document.createElement("span"); - span.textContent = token; - if (parentIsCode) { - // Special handling for tags - span.style.whiteSpace = "pre-wrap"; - span.style.display = "inline"; - } - span.className = - token.trim().length === 0 - ? "stagehand-space" - : "stagehand-highlighted-word"; - fragment.appendChild(span); - }); - - if (fragment.childNodes.length > 0 && node.parentNode) { - element.insertBefore(fragment, node); - node.remove(); - } - } - }); - }); - } - - if (!xpath) { - applyHighlighting(document); - - document.querySelectorAll("iframe").forEach((iframe) => { - try { - iframe.contentWindow?.postMessage({ action: "highlight" }, "*"); - } catch (error) { - console.error("Error accessing iframe content: ", error); - } - }); - } else { - const node = getNodeFromXpath(xpath) as HTMLElement | null; - - if (!node) { - console.warn( - `createTextBoundingBoxes: No element found for xpath "${xpath}".`, - ); - return; - } - - applyHighlighting(node); - } -} - -export function getElementBoundingBoxes(xpath: string): Array<{ - text: string; - top: number; - left: number; - width: number; - height: number; -}> { - const element = getNodeFromXpath(xpath) as HTMLElement; - - if (!element) return []; - - const isValidText = (text: string) => text && text.trim().length > 0; - let dropDownElem = element.querySelector("option[selected]"); - - if (!dropDownElem) { - dropDownElem = element.querySelector("option"); - } - - if (dropDownElem) { - const elemText = dropDownElem.textContent || ""; - if (isValidText(elemText)) { - const parentRect = element.getBoundingClientRect(); - return [ - { - text: elemText.trim(), - top: parentRect.top + window.scrollY, - left: parentRect.left + window.scrollX, - width: parentRect.width, - height: parentRect.height, - }, - ]; - } else { - return []; - } - } - - let placeholderText = ""; - if ( - (element.tagName.toLowerCase() === "input" || - element.tagName.toLowerCase() === "textarea") && - (element as HTMLInputElement).placeholder - ) { - placeholderText = (element as HTMLInputElement).placeholder; - } else if (element.tagName.toLowerCase() === "a") { - placeholderText = ""; - } else if (element.tagName.toLowerCase() === "img") { - placeholderText = (element as HTMLImageElement).alt || ""; - } - - const words = element.querySelectorAll( - ".stagehand-highlighted-word", - ) as NodeListOf; - - const boundingBoxes = Array.from(words) - .map((word) => { - const rect = word.getBoundingClientRect(); - return { - text: word.innerText || "", - top: rect.top + window.scrollY, - left: rect.left + window.scrollX, - width: rect.width, - height: rect.height * 0.75, - }; - }) - .filter( - (box) => - box.width > 0 && - box.height > 0 && - box.top >= 0 && - box.left >= 0 && - isValidText(box.text), - ); - - if (boundingBoxes.length === 0) { - const elementRect = element.getBoundingClientRect(); - return [ - { - text: placeholderText, - top: elementRect.top + window.scrollY, - left: elementRect.left + window.scrollX, - width: elementRect.width, - height: elementRect.height * 0.75, - }, - ]; - } - - return boundingBoxes; -} - -window.waitForDomSettle = waitForDomSettle; -window.processDom = processDom; -window.processAllOfDom = processAllOfDom; -window.storeDOM = storeDOM; -window.restoreDOM = restoreDOM; -window.createTextBoundingBoxes = createTextBoundingBoxes; -window.getElementBoundingBoxes = getElementBoundingBoxes; -window.createStagehandContainer = createStagehandContainer; -window.getScrollableElementXpaths = getScrollableElementXpaths; -window.getNodeFromXpath = getNodeFromXpath; -window.waitForElementScrollEnd = waitForElementScrollEnd; - -async function pickChunk(chunksSeen: Array) { - const viewportHeight = calculateViewportHeight(); - const documentHeight = document.documentElement.scrollHeight; - - const chunks = Math.ceil(documentHeight / viewportHeight); - - const chunksArray = Array.from({ length: chunks }, (_, i) => i); - const chunksRemaining = chunksArray.filter((chunk) => { - return !chunksSeen.includes(chunk); - }); - - const currentScrollPosition = window.scrollY; - const closestChunk = chunksRemaining.reduce((closest, current) => { - const currentChunkTop = viewportHeight * current; - const closestChunkTop = viewportHeight * closest; - return Math.abs(currentScrollPosition - currentChunkTop) < - Math.abs(currentScrollPosition - closestChunkTop) - ? current - : closest; - }, chunksRemaining[0]); - const chunk = closestChunk; - - if (chunk === undefined) { - throw new StagehandDomProcessError( - `No chunks remaining to check: ${chunksRemaining}`, - ); - } - return { - chunk, - chunksArray, - }; -} diff --git a/lib/dom/utils.ts b/lib/dom/utils.ts deleted file mode 100644 index 65255578c..000000000 --- a/lib/dom/utils.ts +++ /dev/null @@ -1,91 +0,0 @@ -import { StagehandDomProcessError } from "@/types/stagehandErrors"; - -export async function waitForDomSettle() { - return new Promise((resolve) => { - const createTimeout = () => { - return setTimeout(() => { - resolve(); - }, 2000); - }; - let timeout = createTimeout(); - const observer = new MutationObserver(() => { - clearTimeout(timeout); - timeout = createTimeout(); - }); - observer.observe(window.document.body, { childList: true, subtree: true }); - }); -} - -export function calculateViewportHeight() { - return Math.ceil(window.innerHeight * 0.75); -} - -/** - * Tests if the element actually responds to .scrollTo(...) - * and that scrollTop changes as expected. - */ -export function canElementScroll(elem: HTMLElement): boolean { - // Quick check if scrollTo is a function - if (typeof elem.scrollTo !== "function") { - console.warn("canElementScroll: .scrollTo is not a function."); - return false; - } - - try { - const originalTop = elem.scrollTop; - - // try to scroll - elem.scrollTo({ - top: originalTop + 100, - left: 0, - behavior: "instant", - }); - - // If scrollTop never changed, consider it unscrollable - if (elem.scrollTop === originalTop) { - throw new StagehandDomProcessError("scrollTop did not change"); - } - - // Scroll back to original place - elem.scrollTo({ - top: originalTop, - left: 0, - behavior: "instant", - }); - - return true; - } catch (error) { - console.warn("canElementScroll error:", (error as Error).message || error); - return false; - } -} - -export function getNodeFromXpath(xpath: string) { - return document.evaluate( - xpath, - document.documentElement, - null, - XPathResult.FIRST_ORDERED_NODE_TYPE, - null, - ).singleNodeValue; -} - -export function waitForElementScrollEnd( - element: HTMLElement, - idleMs = 100, -): Promise { - return new Promise((resolve) => { - let scrollEndTimer: number | undefined; - - const handleScroll = () => { - clearTimeout(scrollEndTimer); - scrollEndTimer = window.setTimeout(() => { - element.removeEventListener("scroll", handleScroll); - resolve(); - }, idleMs); - }; - - element.addEventListener("scroll", handleScroll, { passive: true }); - handleScroll(); - }); -} diff --git a/lib/dom/xpathUtils.ts b/lib/dom/xpathUtils.ts deleted file mode 100644 index f08eb7479..000000000 --- a/lib/dom/xpathUtils.ts +++ /dev/null @@ -1,243 +0,0 @@ -import { isElementNode, isTextNode } from "./elementCheckUtils"; - -function getParentElement(node: ChildNode): Element | null { - return isElementNode(node) - ? node.parentElement - : (node.parentNode as Element); -} - -/** - * Generates all possible combinations of a given array of attributes. - * @param attributes Array of attributes. - * @param size The size of each combination. - * @returns An array of attribute combinations. - */ -function getCombinations( - attributes: { attr: string; value: string }[], - size: number, -): { attr: string; value: string }[][] { - const results: { attr: string; value: string }[][] = []; - - function helper(start: number, combo: { attr: string; value: string }[]) { - if (combo.length === size) { - results.push([...combo]); - return; - } - for (let i = start; i < attributes.length; i++) { - combo.push(attributes[i]); - helper(i + 1, combo); - combo.pop(); - } - } - - helper(0, []); - return results; -} - -/** - * Checks if the generated XPath uniquely identifies the target element. - * @param xpath The XPath string to test. - * @param target The target DOM element. - * @returns True if unique, else false. - */ -function isXPathFirstResultElement(xpath: string, target: Element): boolean { - try { - const result = document.evaluate( - xpath, - document.documentElement, - null, - XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, - null, - ); - return result.snapshotItem(0) === target; - } catch (error) { - // If there's an error evaluating the XPath, consider it not unique - console.warn(`Invalid XPath expression: ${xpath}`, error); - return false; - } -} - -/** - * Escapes a string for use in an XPath expression. - * Handles special characters, including single and double quotes. - * @param value - The string to escape. - * @returns The escaped string safe for XPath. - */ -export function escapeXPathString(value: string): string { - if (value.includes("'")) { - if (value.includes('"')) { - // If the value contains both single and double quotes, split into parts - return ( - "concat(" + - value - .split(/('+)/) - .map((part) => { - if (part === "'") { - return `"'"`; - } else if (part.startsWith("'") && part.endsWith("'")) { - return `"${part}"`; - } else { - return `'${part}'`; - } - }) - .join(",") + - ")" - ); - } else { - // Contains single quotes but not double quotes; use double quotes - return `"${value}"`; - } - } else { - // Does not contain single quotes; use single quotes - return `'${value}'`; - } -} - -/** - * Generates both a complicated XPath and a standard XPath for a given DOM element. - * @param element - The target DOM element. - * @param documentOverride - Optional document override. - * @returns An object containing both XPaths. - */ -export async function generateXPathsForElement( - element: ChildNode, -): Promise { - // Generate the standard XPath - if (!element) return []; - const [complexXPath, standardXPath, idBasedXPath] = await Promise.all([ - generateComplexXPath(element), - generateStandardXPath(element), - generatedIdBasedXPath(element), - ]); - - // This should return in order from most accurate on current page to most cachable. - return [standardXPath, ...(idBasedXPath ? [idBasedXPath] : []), complexXPath]; -} - -async function generateComplexXPath(element: ChildNode): Promise { - // Generate the complicated XPath - const parts: string[] = []; - let currentElement: ChildNode | null = element; - - while ( - currentElement && - (isTextNode(currentElement) || isElementNode(currentElement)) - ) { - if (isElementNode(currentElement)) { - const el = currentElement as Element; - let selector = el.tagName.toLowerCase(); - - // List of attributes to consider for uniqueness - const attributePriority = [ - "data-qa", - "data-component", - "data-role", - "role", - "aria-role", - "type", - "name", - "aria-label", - "placeholder", - "title", - "alt", - ]; - - // Collect attributes present on the element - const attributes = attributePriority - .map((attr) => { - let value = el.getAttribute(attr); - if (attr === "href-full" && value) { - value = el.getAttribute("href"); - } - return value - ? { attr: attr === "href-full" ? "href" : attr, value } - : null; - }) - .filter((attr) => attr !== null) as { attr: string; value: string }[]; - - // Attempt to find a combination of attributes that uniquely identifies the element - let uniqueSelector = ""; - for (let i = 1; i <= attributes.length; i++) { - const combinations = getCombinations(attributes, i); - for (const combo of combinations) { - const conditions = combo - .map((a) => `@${a.attr}=${escapeXPathString(a.value)}`) - .join(" and "); - const xpath = `//${selector}[${conditions}]`; - if (isXPathFirstResultElement(xpath, el)) { - uniqueSelector = xpath; - break; - } - } - if (uniqueSelector) break; - } - - if (uniqueSelector) { - parts.unshift(uniqueSelector.replace("//", "")); - break; - } else { - // Fallback to positional selector - const parent = getParentElement(el); - if (parent) { - const siblings = Array.from(parent.children).filter( - (sibling) => sibling.tagName === el.tagName, - ); - const index = siblings.indexOf(el as HTMLElement) + 1; - selector += siblings.length > 1 ? `[${index}]` : ""; - } - parts.unshift(selector); - } - } - - currentElement = getParentElement(currentElement); - } - - const xpath = "//" + parts.join("/"); - return xpath; -} - -/** - * Generates a standard XPath for a given DOM element. - * @param element - The target DOM element. - * @returns A standard XPath string. - */ -async function generateStandardXPath(element: ChildNode): Promise { - const parts: string[] = []; - while (element && (isTextNode(element) || isElementNode(element))) { - let index = 0; - let hasSameTypeSiblings = false; - const siblings = element.parentElement - ? Array.from(element.parentElement.childNodes) - : []; - for (let i = 0; i < siblings.length; i++) { - const sibling = siblings[i]; - if ( - sibling.nodeType === element.nodeType && - sibling.nodeName === element.nodeName - ) { - index = index + 1; - hasSameTypeSiblings = true; - if (sibling.isSameNode(element)) { - break; - } - } - } - // text "nodes" are selected differently than elements with xPaths - if (element.nodeName !== "#text") { - const tagName = element.nodeName.toLowerCase(); - const pathIndex = hasSameTypeSiblings ? `[${index}]` : ""; - parts.unshift(`${tagName}${pathIndex}`); - } - element = element.parentElement as HTMLElement; - } - return parts.length ? `/${parts.join("/")}` : ""; -} - -async function generatedIdBasedXPath( - element: ChildNode, -): Promise { - if (isElementNode(element) && element.id) { - return `//*[@id='${element.id}']`; - } - return null; -} diff --git a/lib/handlers/actHandler.ts b/lib/handlers/actHandler.ts deleted file mode 100644 index d3442f07e..000000000 --- a/lib/handlers/actHandler.ts +++ /dev/null @@ -1,356 +0,0 @@ -import { Locator } from "@playwright/test"; -import { LogLine } from "../../types/log"; -import { - PlaywrightCommandException, - PlaywrightCommandMethodNotSupportedException, -} from "../../types/playwright"; -import { LLMClient } from "../llm/LLMClient"; -import { StagehandPage } from "../StagehandPage"; -import { - ActResult, - ObserveResult, - ActOptions, - ObserveOptions, -} from "@/types/stagehand"; -import { MethodHandlerContext, SupportedPlaywrightAction } from "@/types/act"; -import { buildActObservePrompt } from "../prompt"; -import { - methodHandlerMap, - fallbackLocatorMethod, -} from "./handlerUtils/actHandlerUtils"; -import { StagehandObserveHandler } from "@/lib/handlers/observeHandler"; -import { StagehandInvalidArgumentError } from "@/types/stagehandErrors"; -/** - * NOTE: Vision support has been removed from this version of Stagehand. - * If useVision or verifierUseVision is set to true, a warning is logged and - * the flow continues as if vision = false. - */ -export class StagehandActHandler { - private readonly stagehandPage: StagehandPage; - private readonly logger: (logLine: LogLine) => void; - private readonly selfHeal: boolean; - - constructor({ - logger, - stagehandPage, - selfHeal, - }: { - logger: (logLine: LogLine) => void; - stagehandPage: StagehandPage; - selfHeal: boolean; - }) { - this.logger = logger; - this.stagehandPage = stagehandPage; - this.selfHeal = selfHeal; - } - - /** - * Perform an immediate Playwright action based on an ObserveResult object - * that was returned from `page.observe(...)`. - */ - public async actFromObserveResult( - observe: ObserveResult, - domSettleTimeoutMs?: number, - ): Promise { - this.logger({ - category: "action", - message: "Performing act from an ObserveResult", - level: 1, - auxiliary: { - observeResult: { - value: JSON.stringify(observe), - type: "object", - }, - }, - }); - - const method = observe.method; - if (method === "not-supported") { - this.logger({ - category: "action", - message: "Cannot execute ObserveResult with unsupported method", - level: 1, - auxiliary: { - error: { - value: - "NotSupportedError: The method requested in this ObserveResult is not supported by Stagehand.", - type: "string", - }, - trace: { - value: `Cannot execute act from ObserveResult with unsupported method: ${method}`, - type: "string", - }, - }, - }); - return { - success: false, - message: `Unable to perform action: The method '${method}' is not supported in ObserveResult. Please use a supported Playwright locator method.`, - action: observe.description || `ObserveResult action (${method})`, - }; - } - const args = observe.arguments ?? []; - // remove the xpath prefix on the selector - const selector = observe.selector.replace("xpath=", ""); - - try { - await this._performPlaywrightMethod( - method, - args, - selector, - domSettleTimeoutMs, - ); - - return { - success: true, - message: `Action [${method}] performed successfully on selector: ${selector}`, - action: observe.description || `ObserveResult action (${method})`, - }; - } catch (err) { - if ( - !this.selfHeal || - err instanceof PlaywrightCommandMethodNotSupportedException - ) { - this.logger({ - category: "action", - message: "Error performing act from an ObserveResult", - level: 1, - auxiliary: { - error: { value: err.message, type: "string" }, - trace: { value: err.stack, type: "string" }, - }, - }); - return { - success: false, - message: `Failed to perform act: ${err.message}`, - action: observe.description || `ObserveResult action (${method})`, - }; - } - // We will try to use observeAct on a failed ObserveResult-act if selfHeal is true - this.logger({ - category: "action", - message: - "Error performing act from an ObserveResult. Reprocessing the page and trying again", - level: 1, - auxiliary: { - error: { value: err.message, type: "string" }, - trace: { value: err.stack, type: "string" }, - observeResult: { value: JSON.stringify(observe), type: "object" }, - }, - }); - try { - // Remove redundancy from method-description - const actCommand = observe.description - .toLowerCase() - .startsWith(method.toLowerCase()) - ? observe.description - : method - ? `${method} ${observe.description}` - : observe.description; - // Call act with the ObserveResult description - return await this.stagehandPage.act({ - action: actCommand, - }); - } catch (err) { - this.logger({ - category: "action", - message: "Error performing act from an ObserveResult on fallback", - level: 1, - auxiliary: { - error: { value: err.message, type: "string" }, - trace: { value: err.stack, type: "string" }, - }, - }); - return { - success: false, - message: `Failed to perform act: ${err.message}`, - action: observe.description || `ObserveResult action (${method})`, - }; - } - } - } - - /** - * Perform an act based on an instruction. - * This method will observe the page and then perform the act on the first element returned. - */ - public async observeAct( - actionOrOptions: ActOptions, - observeHandler: StagehandObserveHandler, - llmClient: LLMClient, - requestId: string, - ): Promise { - // Extract the action string - let action: string; - const observeOptions: Partial = {}; - - if (typeof actionOrOptions === "object" && actionOrOptions !== null) { - if (!("action" in actionOrOptions)) { - throw new StagehandInvalidArgumentError( - "Invalid argument. Action options must have an `action` field.", - ); - } - - if ( - typeof actionOrOptions.action !== "string" || - actionOrOptions.action.length === 0 - ) { - throw new StagehandInvalidArgumentError( - "Invalid argument. No action provided.", - ); - } - - action = actionOrOptions.action; - - // Extract options that should be passed to observe - if (actionOrOptions.modelName) - observeOptions.modelName = actionOrOptions.modelName; - if (actionOrOptions.modelClientOptions) - observeOptions.modelClientOptions = actionOrOptions.modelClientOptions; - } else { - throw new StagehandInvalidArgumentError( - "Invalid argument. Valid arguments are: a string, an ActOptions object with an `action` field not empty, or an ObserveResult with a `selector` and `method` field.", - ); - } - - // doObserveAndAct is just a wrapper of observeAct and actFromObserveResult. - // we did this so that we can cleanly call a Promise.race, and race - // doObserveAndAct against the user defined timeoutMs (if one was defined) - const doObserveAndAct = async (): Promise => { - const instruction = buildActObservePrompt( - action, - Object.values(SupportedPlaywrightAction), - actionOrOptions.variables, - ); - - const observeResults = await observeHandler.observe({ - instruction, - llmClient, - requestId, - onlyVisible: false, - drawOverlay: false, - returnAction: true, - fromAct: true, - }); - - if (observeResults.length === 0) { - return { - success: false, - message: `Failed to perform act: No observe results found for action`, - action, - }; - } - - const element: ObserveResult = observeResults[0]; - - if (actionOrOptions.variables) { - Object.keys(actionOrOptions.variables).forEach((key) => { - element.arguments = element.arguments.map((arg) => - arg.replace(`%${key}%`, actionOrOptions.variables![key]), - ); - }); - } - - return this.actFromObserveResult( - element, - actionOrOptions.domSettleTimeoutMs, - ); - }; - - // if no user defined timeoutMs, just do observeAct + actFromObserveResult - // with no timeout - if (!actionOrOptions.timeoutMs) { - return doObserveAndAct(); - } - - // Race observeAct + actFromObserveResult vs. the timeoutMs - const { timeoutMs } = actionOrOptions; - return await Promise.race([ - doObserveAndAct(), - new Promise((resolve) => { - setTimeout(() => { - resolve({ - success: false, - message: `Action timed out after ${timeoutMs}ms`, - action, - }); - }, timeoutMs); - }), - ]); - } - - private async _performPlaywrightMethod( - method: string, - args: unknown[], - xpath: string, - domSettleTimeoutMs?: number, - ) { - const locator = this.stagehandPage.page.locator(`xpath=${xpath}`).first(); - const initialUrl = this.stagehandPage.page.url(); - - this.logger({ - category: "action", - message: "performing playwright method", - level: 2, - auxiliary: { - xpath: { value: xpath, type: "string" }, - method: { value: method, type: "string" }, - }, - }); - - const context: MethodHandlerContext = { - method, - locator, - xpath, - args, - logger: this.logger, - stagehandPage: this.stagehandPage, - initialUrl, - domSettleTimeoutMs, - }; - - try { - // 1) Look up a function in the map - const methodFn = methodHandlerMap[method]; - - // 2) If found, call it - if (methodFn) { - await methodFn(context); - - // 3) Otherwise, see if it's a valid locator method - } else if (typeof locator[method as keyof Locator] === "function") { - await fallbackLocatorMethod(context); - - // 4) If still unknown, we can’t handle it - } else { - this.logger({ - category: "action", - message: "chosen method is invalid", - level: 1, - auxiliary: { - method: { value: method, type: "string" }, - }, - }); - throw new PlaywrightCommandMethodNotSupportedException( - `Method ${method} not supported`, - ); - } - - // Always wait for DOM to settle - await this.stagehandPage._waitForSettledDom(domSettleTimeoutMs); - } catch (e) { - this.logger({ - category: "action", - message: "error performing method", - level: 1, - auxiliary: { - error: { value: e.message, type: "string" }, - trace: { value: e.stack, type: "string" }, - method: { value: method, type: "string" }, - xpath: { value: xpath, type: "string" }, - args: { value: JSON.stringify(args), type: "object" }, - }, - }); - throw new PlaywrightCommandException(e.message); - } - } -} diff --git a/lib/handlers/agentHandler.ts b/lib/handlers/agentHandler.ts deleted file mode 100644 index 3bb0e0227..000000000 --- a/lib/handlers/agentHandler.ts +++ /dev/null @@ -1,671 +0,0 @@ -import { StagehandPage } from "../StagehandPage"; -import { AgentProvider } from "../agent/AgentProvider"; -import { StagehandAgent } from "../agent/StagehandAgent"; -import { AgentClient } from "../agent/AgentClient"; -import { LogLine } from "../../types/log"; -import { Page } from "playwright"; -import { - AgentExecuteOptions, - AgentAction, - AgentResult, - AgentHandlerOptions, - ActionExecutionResult, -} from "@/types/agent"; - -export class StagehandAgentHandler { - private stagehandPage: StagehandPage; - private agent: StagehandAgent; - private provider: AgentProvider; - private logger: (message: LogLine) => void; - private agentClient: AgentClient; - private options: AgentHandlerOptions; - - constructor( - stagehandPage: StagehandPage, - logger: (message: LogLine) => void, - options: AgentHandlerOptions, - ) { - this.stagehandPage = stagehandPage; - this.logger = logger; - this.options = options; - - // Initialize the provider - this.provider = new AgentProvider(logger); - - // Create client first - const client = this.provider.getClient( - options.modelName, - options.clientOptions || {}, - options.userProvidedInstructions, - ); - - // Store the client - this.agentClient = client; - - // Set up common functionality for any client type - this.setupAgentClient(); - - // Create agent with the client - this.agent = new StagehandAgent(client, logger); - } - - private setupAgentClient(): void { - // Set up screenshot provider for any client type - this.agentClient.setScreenshotProvider(async () => { - const screenshot = await this.stagehandPage.page.screenshot({ - fullPage: false, - }); - // Convert to base64 - return screenshot.toString("base64"); - }); - - // Set up action handler for any client type - this.agentClient.setActionHandler(async (action) => { - // Default delay between actions (1 second if not specified) - const defaultDelay = 1000; - // Use specified delay or default - const waitBetweenActions = - (this.options.clientOptions?.waitBetweenActions as number) || - defaultDelay; - - try { - // Try to inject cursor before each action - try { - await this.injectCursor(); - } catch { - // Ignore cursor injection failures - } - - // Add a small delay before the action for better visibility - await new Promise((resolve) => setTimeout(resolve, 500)); - - // Execute the action - await this.executeAction(action); - - // Add a delay after the action for better visibility - await new Promise((resolve) => setTimeout(resolve, waitBetweenActions)); - - // After executing an action, take a screenshot - try { - await this.captureAndSendScreenshot(); - } catch (error) { - const errorMessage = - error instanceof Error ? error.message : String(error); - this.logger({ - category: "agent", - message: `Warning: Failed to take screenshot after action: ${errorMessage}. Continuing execution.`, - level: 1, - }); - // Continue execution even if screenshot fails - } - } catch (error) { - const errorMessage = - error instanceof Error ? error.message : String(error); - this.logger({ - category: "agent", - message: `Error executing action ${action.type}: ${errorMessage}`, - level: 0, - }); - throw error; // Re-throw the error to be handled by the caller - } - }); - - // Update viewport and URL for any client type - this.updateClientViewport(); - this.updateClientUrl(); - } - - /** - * Execute a task with the agent - */ - async execute( - optionsOrInstruction: AgentExecuteOptions | string, - ): Promise { - const options = - typeof optionsOrInstruction === "string" - ? { instruction: optionsOrInstruction } - : optionsOrInstruction; - - //Redirect to Google if the URL is empty or about:blank - const currentUrl = this.stagehandPage.page.url(); - if (!currentUrl || currentUrl === "about:blank") { - this.logger({ - category: "agent", - message: `Page URL is empty or about:blank. Redirecting to www.google.com...`, - level: 0, - }); - await this.stagehandPage.page.goto("https://www.google.com"); - } - - this.logger({ - category: "agent", - message: `Executing agent task: ${options.instruction}`, - level: 1, - }); - - // Inject cursor for visual feedback - try { - await this.injectCursor(); - } catch (error) { - const errorMessage = - error instanceof Error ? error.message : String(error); - this.logger({ - category: "agent", - message: `Warning: Failed to inject cursor: ${errorMessage}. Continuing with execution.`, - level: 1, - }); - // Continue execution even if cursor injection fails - } - - // Take initial screenshot if needed - if (options.autoScreenshot !== false) { - try { - await this.captureAndSendScreenshot(); - } catch (error) { - const errorMessage = - error instanceof Error ? error.message : String(error); - this.logger({ - category: "agent", - message: `Warning: Failed to take initial screenshot: ${errorMessage}. Continuing with execution.`, - level: 1, - }); - // Continue execution even if screenshot fails - } - } - - // Execute the task - const result = await this.agent.execute(optionsOrInstruction); - - // The actions are now executed during the agent's execution flow - // We don't need to execute them again here - - return result; - } - - /** - * Execute a single action on the page - */ - private async executeAction( - action: AgentAction, - ): Promise { - try { - switch (action.type) { - case "click": { - const { x, y, button = "left" } = action; - // Update cursor position first - await this.updateCursorPosition(x as number, y as number); - // Animate the click - await this.animateClick(x as number, y as number); - // Small delay to see the animation - await new Promise((resolve) => setTimeout(resolve, 300)); - // Perform the actual click - await this.stagehandPage.page.mouse.click(x as number, y as number, { - button: button as "left" | "right", - }); - const newOpenedTab = await Promise.race([ - new Promise((resolve) => { - this.stagehandPage.context.once("page", (page) => resolve(page)); - setTimeout(() => resolve(null), 1500); - }), - ]); - if (newOpenedTab) { - this.logger({ - category: "action", - message: `New page detected (new tab) with URL. Opening on current page...`, - level: 1, - auxiliary: { - url: { - value: newOpenedTab.url(), - type: "string", - }, - }, - }); - await newOpenedTab.close(); - await this.stagehandPage.page.goto(newOpenedTab.url()); - await this.stagehandPage.page.waitForURL(newOpenedTab.url()); - } - return { success: true }; - } - - case "double_click": { - const { x, y } = action; - // Update cursor position first - await this.updateCursorPosition(x as number, y as number); - // Animate the click - await this.animateClick(x as number, y as number); - // Small delay to see the animation - await new Promise((resolve) => setTimeout(resolve, 200)); - // Animate the second click - await this.animateClick(x as number, y as number); - // Small delay to see the animation - await new Promise((resolve) => setTimeout(resolve, 200)); - // Perform the actual double click - await this.stagehandPage.page.mouse.dblclick( - x as number, - y as number, - ); - return { success: true }; - } - - // Handle the case for "doubleClick" as well for backward compatibility - case "doubleClick": { - const { x, y } = action; - // Update cursor position first - await this.updateCursorPosition(x as number, y as number); - // Animate the click - await this.animateClick(x as number, y as number); - // Small delay to see the animation - await new Promise((resolve) => setTimeout(resolve, 200)); - // Animate the second click - await this.animateClick(x as number, y as number); - // Small delay to see the animation - await new Promise((resolve) => setTimeout(resolve, 200)); - // Perform the actual double click - await this.stagehandPage.page.mouse.dblclick( - x as number, - y as number, - ); - return { success: true }; - } - - case "type": { - const { text } = action; - await this.stagehandPage.page.keyboard.type(text as string); - return { success: true }; - } - - case "keypress": { - const { keys } = action; - if (Array.isArray(keys)) { - for (const key of keys) { - // Handle special keys - if (key.includes("ENTER")) { - await this.stagehandPage.page.keyboard.press("Enter"); - } else if (key.includes("SPACE")) { - await this.stagehandPage.page.keyboard.press(" "); - } else if (key.includes("TAB")) { - await this.stagehandPage.page.keyboard.press("Tab"); - } else if (key.includes("ESCAPE") || key.includes("ESC")) { - await this.stagehandPage.page.keyboard.press("Escape"); - } else if (key.includes("BACKSPACE")) { - await this.stagehandPage.page.keyboard.press("Backspace"); - } else if (key.includes("DELETE")) { - await this.stagehandPage.page.keyboard.press("Delete"); - } else if (key.includes("ARROW_UP")) { - await this.stagehandPage.page.keyboard.press("ArrowUp"); - } else if (key.includes("ARROW_DOWN")) { - await this.stagehandPage.page.keyboard.press("ArrowDown"); - } else if (key.includes("ARROW_LEFT")) { - await this.stagehandPage.page.keyboard.press("ArrowLeft"); - } else if (key.includes("ARROW_RIGHT")) { - await this.stagehandPage.page.keyboard.press("ArrowRight"); - } else { - // For other keys, use the existing conversion - const playwrightKey = this.convertKeyName(key); - await this.stagehandPage.page.keyboard.press(playwrightKey); - } - } - } - return { success: true }; - } - - case "scroll": { - const { x, y, scroll_x = 0, scroll_y = 0 } = action; - // First move to the position - await this.stagehandPage.page.mouse.move(x as number, y as number); - // Then scroll - await this.stagehandPage.page.evaluate( - ({ scrollX, scrollY }) => window.scrollBy(scrollX, scrollY), - { scrollX: scroll_x as number, scrollY: scroll_y as number }, - ); - return { success: true }; - } - - case "drag": { - const { path } = action; - if (Array.isArray(path) && path.length >= 2) { - const start = path[0]; - - // Update cursor position for start - await this.updateCursorPosition(start.x, start.y); - await this.stagehandPage.page.mouse.move(start.x, start.y); - await this.stagehandPage.page.mouse.down(); - - // Update cursor position for each point in the path - for (let i = 1; i < path.length; i++) { - await this.updateCursorPosition(path[i].x, path[i].y); - await this.stagehandPage.page.mouse.move(path[i].x, path[i].y); - } - - await this.stagehandPage.page.mouse.up(); - } - return { success: true }; - } - - case "move": { - const { x, y } = action; - // Update cursor position first - await this.updateCursorPosition(x as number, y as number); - await this.stagehandPage.page.mouse.move(x as number, y as number); - return { success: true }; - } - - case "wait": { - await new Promise((resolve) => setTimeout(resolve, 1000)); - return { success: true }; - } - - case "screenshot": { - // Screenshot is handled automatically by the agent client - // after each action, so we don't need to do anything here - return { success: true }; - } - - case "function": { - const { name, arguments: args = {} } = action; - - if ( - name === "goto" && - typeof args === "object" && - args !== null && - "url" in args - ) { - await this.stagehandPage.page.goto(args.url as string); - this.updateClientUrl(); - return { success: true }; - } else if (name === "back") { - await this.stagehandPage.page.goBack(); - this.updateClientUrl(); - return { success: true }; - } else if (name === "forward") { - await this.stagehandPage.page.goForward(); - this.updateClientUrl(); - return { success: true }; - } else if (name === "reload") { - await this.stagehandPage.page.reload(); - this.updateClientUrl(); - return { success: true }; - } - - return { - success: false, - error: `Unsupported function: ${name}`, - }; - } - - case "key": { - // Handle the 'key' action type from Anthropic - const { text } = action; - if (text === "Return" || text === "Enter") { - await this.stagehandPage.page.keyboard.press("Enter"); - } else if (text === "Tab") { - await this.stagehandPage.page.keyboard.press("Tab"); - } else if (text === "Escape" || text === "Esc") { - await this.stagehandPage.page.keyboard.press("Escape"); - } else if (text === "Backspace") { - await this.stagehandPage.page.keyboard.press("Backspace"); - } else { - // For other keys, try to press directly - await this.stagehandPage.page.keyboard.press(text as string); - } - return { success: true }; - } - - default: - return { - success: false, - error: `Unsupported action type: ${action.type}`, - }; - } - } catch (error) { - const errorMessage = - error instanceof Error ? error.message : String(error); - - this.logger({ - category: "agent", - message: `Error executing action ${action.type}: ${errorMessage}`, - level: 0, - }); - - return { - success: false, - error: errorMessage, - }; - } - } - - private updateClientViewport(): void { - const viewportSize = this.stagehandPage.page.viewportSize(); - if (viewportSize) { - this.agentClient.setViewport(viewportSize.width, viewportSize.height); - } - } - - private updateClientUrl(): void { - const url = this.stagehandPage.page.url(); - this.agentClient.setCurrentUrl(url); - } - - getAgent(): StagehandAgent { - return this.agent; - } - - getClient(): AgentClient { - return this.agentClient; - } - - async captureAndSendScreenshot(): Promise { - this.logger({ - category: "agent", - message: "Taking screenshot and sending to agent", - level: 1, - }); - - try { - // Take screenshot of the current page - const screenshot = await this.stagehandPage.page.screenshot({ - type: "png", - fullPage: false, - }); - - // Convert to base64 - const base64Image = screenshot.toString("base64"); - - // Just use the captureScreenshot method on the agent client - return await this.agentClient.captureScreenshot({ - base64Image, - currentUrl: this.stagehandPage.page.url(), - }); - } catch (error) { - const errorMessage = - error instanceof Error ? error.message : String(error); - this.logger({ - category: "agent", - message: `Error capturing screenshot: ${errorMessage}`, - level: 0, - }); - return null; - } - } - - /** - * Inject a cursor element into the page for visual feedback - */ - private async injectCursor(): Promise { - try { - // Define constants for cursor and highlight element IDs - const CURSOR_ID = "stagehand-cursor"; - const HIGHLIGHT_ID = "stagehand-highlight"; - - // Check if cursor already exists - const cursorExists = await this.stagehandPage.page.evaluate( - (id: string) => { - return !!document.getElementById(id); - }, - CURSOR_ID, - ); - - if (cursorExists) { - return; - } - - // Inject cursor and highlight elements - await this.stagehandPage.page.evaluate(` - (function(cursorId, highlightId) { - // Create cursor element - const cursor = document.createElement('div'); - cursor.id = cursorId; - - // Use the provided SVG for a custom cursor - cursor.innerHTML = \` - - - - - \`; - - // Style the cursor - cursor.style.position = 'absolute'; - cursor.style.top = '0'; - cursor.style.left = '0'; - cursor.style.width = '28px'; - cursor.style.height = '28px'; - cursor.style.pointerEvents = 'none'; - cursor.style.zIndex = '9999999'; - cursor.style.transform = 'translate(-4px, -4px)'; // Adjust to align the pointer tip - - // Create highlight element for click animation - const highlight = document.createElement('div'); - highlight.id = highlightId; - highlight.style.position = 'absolute'; - highlight.style.width = '20px'; - highlight.style.height = '20px'; - highlight.style.borderRadius = '50%'; - highlight.style.backgroundColor = 'rgba(66, 134, 244, 0)'; - highlight.style.transform = 'translate(-50%, -50%) scale(0)'; - highlight.style.pointerEvents = 'none'; - highlight.style.zIndex = '9999998'; - highlight.style.transition = 'transform 0.3s ease-out, opacity 0.3s ease-out'; - highlight.style.opacity = '0'; - - // Add elements to the document - document.body.appendChild(cursor); - document.body.appendChild(highlight); - - // Add a function to update cursor position - window.__updateCursorPosition = function(x, y) { - if (cursor) { - cursor.style.transform = \`translate(\${x - 4}px, \${y - 4}px)\`; - } - }; - - // Add a function to animate click - window.__animateClick = function(x, y) { - if (highlight) { - highlight.style.left = \`\${x}px\`; - highlight.style.top = \`\${y}px\`; - highlight.style.transform = 'translate(-50%, -50%) scale(1)'; - highlight.style.opacity = '1'; - - setTimeout(() => { - highlight.style.transform = 'translate(-50%, -50%) scale(0)'; - highlight.style.opacity = '0'; - }, 300); - } - }; - })('${CURSOR_ID}', '${HIGHLIGHT_ID}'); - `); - - this.logger({ - category: "agent", - message: "Cursor injected for visual feedback", - level: 1, - }); - } catch (error) { - this.logger({ - category: "agent", - message: `Failed to inject cursor: ${error}`, - level: 0, - }); - } - } - - /** - * Update the cursor position on the page - */ - private async updateCursorPosition(x: number, y: number): Promise { - try { - await this.stagehandPage.page.evaluate( - ({ x, y }) => { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - if ((window as any).__updateCursorPosition) { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - (window as any).__updateCursorPosition(x, y); - } - }, - { x, y }, - ); - } catch { - // Silently fail if cursor update fails - // This is not critical functionality - } - } - - /** - * Animate a click at the given position - */ - private async animateClick(x: number, y: number): Promise { - try { - await this.stagehandPage.page.evaluate( - ({ x, y }) => { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - if ((window as any).__animateClick) { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - (window as any).__animateClick(x, y); - } - }, - { x, y }, - ); - } catch { - // Silently fail if animation fails - // This is not critical functionality - } - } - - private convertKeyName(key: string): string { - // Map of CUA key names to Playwright key names - const keyMap: Record = { - ENTER: "Enter", - ESCAPE: "Escape", - BACKSPACE: "Backspace", - TAB: "Tab", - SPACE: " ", - ARROWUP: "ArrowUp", - ARROWDOWN: "ArrowDown", - ARROWLEFT: "ArrowLeft", - ARROWRIGHT: "ArrowRight", - UP: "ArrowUp", - DOWN: "ArrowDown", - LEFT: "ArrowLeft", - RIGHT: "ArrowRight", - SHIFT: "Shift", - CONTROL: "Control", - ALT: "Alt", - META: "Meta", - COMMAND: "Meta", - CMD: "Meta", - CTRL: "Control", - DELETE: "Delete", - HOME: "Home", - END: "End", - PAGEUP: "PageUp", - PAGEDOWN: "PageDown", - }; - - // Convert to uppercase for case-insensitive matching - const upperKey = key.toUpperCase(); - - // Return the mapped key or the original key if not found - return keyMap[upperKey] || key; - } -} diff --git a/lib/handlers/extractHandler.ts b/lib/handlers/extractHandler.ts deleted file mode 100644 index 085d09c79..000000000 --- a/lib/handlers/extractHandler.ts +++ /dev/null @@ -1,896 +0,0 @@ -import { z, ZodTypeAny } from "zod"; -import { LogLine } from "../../types/log"; -import { ZodPathSegments } from "../../types/stagehand"; -import { TextAnnotation } from "../../types/textannotation"; -import { extract } from "../inference"; -import { LLMClient } from "../llm/LLMClient"; -import { formatText } from "../utils"; -import { StagehandPage } from "../StagehandPage"; -import { Stagehand, StagehandFunctionName } from "../index"; -import { pageTextSchema } from "../../types/page"; -import { getAccessibilityTree } from "@/lib/a11y/utils"; - -const PROXIMITY_THRESHOLD = 15; - -/** - * The `StagehandExtractHandler` class is responsible for extracting structured data from a webpage. - * It provides two approaches: `textExtract` and `domExtract`. `textExtract` is used by default. - * - * Here is what `textExtract` does at a high level: - * - * **1. Wait for the DOM to settle and start DOM debugging.** - * - Ensures the page is fully loaded and stable before extraction. - * - * **2. Store the original DOM before any mutations.** - * - Preserves the initial state of the DOM to restore later. - * - We do this because creating spans around every word in the DOM (see step 4) - * becomes very difficult to revert. Text nodes can be finicky, and directly - * removing the added spans often corrupts the structure of the DOM. - * - * **3. Process the DOM to generate a selector map of candidate elements.** - * - Identifies potential elements that contain the data to extract. - * - * **4. Create text bounding boxes around every word in the webpage.** - * - Wraps words in spans so that their bounding boxes can be used to - * determine their positions on the text-rendered-webpage. - * - * **5. Collect all text annotations (with positions and dimensions) from each of the candidate elements.** - * - Gathers text and positional data for each word. - * - * **6. Group annotations by text and deduplicate them based on proximity.** - * - There is no guarantee that the text annotations are unique (candidate elements can be nested). - * - Thus, we must remove duplicate words that are close to each other on the page. - * - * **7. Restore the original DOM after mutations.** - * - Returns the DOM to its original state after processing. - * - * **8. Format the deduplicated annotations into a text representation.** - * - Prepares the text data for the extraction process. - * - * **9. Pass the formatted text to an LLM for extraction according to the given instruction and schema.** - * - Uses a language model to extract structured data based on instructions. - * - * **10. Handle the extraction response and logging the results.** - * - Processes the output from the LLM and logs relevant information. - * - * @remarks - * Each step corresponds to specific code segments, as noted in the comments throughout the code. - */ - -export class StagehandExtractHandler { - private readonly stagehand: Stagehand; - private readonly stagehandPage: StagehandPage; - private readonly logger: (logLine: LogLine) => void; - private readonly userProvidedInstructions?: string; - - constructor({ - stagehand, - logger, - stagehandPage, - userProvidedInstructions, - }: { - stagehand: Stagehand; - logger: (message: { - category?: string; - message: string; - level?: number; - auxiliary?: { [key: string]: { value: string; type: string } }; - }) => void; - stagehandPage: StagehandPage; - userProvidedInstructions?: string; - }) { - this.stagehand = stagehand; - this.logger = logger; - this.stagehandPage = stagehandPage; - this.userProvidedInstructions = userProvidedInstructions; - } - - public async extract({ - instruction, - schema, - content = {}, - llmClient, - requestId, - domSettleTimeoutMs, - useTextExtract = false, - selector, - }: { - instruction?: string; - schema?: T; - content?: z.infer; - chunksSeen?: Array; - llmClient?: LLMClient; - requestId?: string; - domSettleTimeoutMs?: number; - useTextExtract?: boolean; - selector?: string; - } = {}): Promise> { - const noArgsCalled = !instruction && !schema && !llmClient && !selector; - if (noArgsCalled) { - this.logger({ - category: "extraction", - message: "Extracting the entire page text.", - level: 1, - }); - return this.extractPageText(); - } - - if (useTextExtract) { - return this.textExtract({ - instruction, - schema, - content, - llmClient, - requestId, - domSettleTimeoutMs, - selector, - }); - } else { - return this.domExtract({ - instruction, - schema, - content, - llmClient, - requestId, - domSettleTimeoutMs, - }); - } - } - - private async extractPageText(): Promise<{ page_text?: string }> { - await this.stagehandPage._waitForSettledDom(); - - const originalDOM = await this.stagehandPage.page.evaluate(() => - window.storeDOM(undefined), - ); - - const { selectorMap }: { selectorMap: Record } = - await this.stagehand.page.evaluate(() => - window.processAllOfDom(undefined), - ); - - await this.stagehand.page.evaluate(() => - window.createTextBoundingBoxes(undefined), - ); - - const containerDims = await this.getTargetDimensions(); - - const allAnnotations = await this.collectAllAnnotations( - selectorMap, - containerDims.width, - containerDims.height, - containerDims.offsetLeft, - containerDims.offsetTop, - ); - - const deduplicatedTextAnnotations = - this.deduplicateAnnotations(allAnnotations); - - await this.stagehandPage.page.evaluate( - (dom) => window.restoreDOM(dom, undefined), - originalDOM, - ); - - const formattedText = formatText( - deduplicatedTextAnnotations, - containerDims.width, - ); - - const result = { page_text: formattedText }; - return pageTextSchema.parse(result); - } - - private async textExtract({ - instruction, - schema, - content = {}, - llmClient, - requestId, - domSettleTimeoutMs, - selector, - }: { - instruction?: string; - schema?: T; - content?: z.infer; - llmClient?: LLMClient; - requestId?: string; - domSettleTimeoutMs?: number; - selector?: string; - }): Promise> { - this.logger({ - category: "extraction", - message: "starting extraction", - level: 1, - auxiliary: { - instruction: { - value: instruction, - type: "string", - }, - }, - }); - - // **1:** Wait for the DOM to settle and start DOM debugging - await this.stagehandPage._waitForSettledDom(domSettleTimeoutMs); - - const targetXpath = selector?.replace(/^xpath=/, "") ?? ""; - - // **2:** Store the original DOM before any mutations - // we need to store the original DOM here because calling createTextBoundingBoxes() - // will mutate the DOM by adding spans around every word - const originalDOM = await this.stagehandPage.page.evaluate( - (xp) => window.storeDOM(xp), - targetXpath, - ); - - // **3:** Process the DOM to generate a selector map of candidate elements - const { selectorMap }: { selectorMap: Record } = - await this.stagehand.page.evaluate( - (xp) => window.processAllOfDom(xp), - targetXpath, - ); - - this.logger({ - category: "extraction", - message: `received output from processAllOfDom. selectorMap has ${ - Object.keys(selectorMap).length - } entries`, - level: 1, - }); - - // **4:** Create text bounding boxes around every word in the webpage - // calling createTextBoundingBoxes() will create a span around every word on the - // webpage. The bounding boxes of these spans will be used to determine their - // positions in the text rendered webpage - await this.stagehand.page.evaluate( - (xp) => window.createTextBoundingBoxes(xp), - targetXpath, - ); - - // **5:** Determine the container dimensions for either the entire page or the target element - const { - width: containerWidth, - height: containerHeight, - offsetLeft = 0, - offsetTop = 0, - } = await this.getTargetDimensions(targetXpath); - - // **6:** Collect all text annotations (with positions and dimensions) from the candidate elements - // allAnnotations will store all the TextAnnotations BEFORE deduplication - const allAnnotations = await this.collectAllAnnotations( - selectorMap, - containerWidth, - containerHeight, - offsetLeft, - offsetTop, - ); - - // **7:** Group annotations by text and deduplicate them based on proximity - const annotationsGroupedByText = new Map(); - - for (const annotation of allAnnotations) { - if (!annotationsGroupedByText.has(annotation.text)) { - annotationsGroupedByText.set(annotation.text, []); - } - annotationsGroupedByText.get(annotation.text)!.push(annotation); - } - - const deduplicatedTextAnnotations: TextAnnotation[] = []; - - // here, we deduplicate annotations per text group - for (const [text, annotations] of annotationsGroupedByText.entries()) { - for (const annotation of annotations) { - // check if this annotation is close to any existing deduplicated annotation - const isDuplicate = deduplicatedTextAnnotations.some( - (existingAnnotation) => { - if (existingAnnotation.text !== text) return false; - - const dx = - existingAnnotation.bottom_left.x - annotation.bottom_left.x; - const dy = - existingAnnotation.bottom_left.y - annotation.bottom_left.y; - const distance = Math.hypot(dx, dy); - // the annotation is a duplicate if it has the same text and its bottom_left - // position is within the PROXIMITY_THRESHOLD of an existing annotation. - // we calculate the Euclidean distance between the two bottom_left points, - // and if the distance is less than PROXIMITY_THRESHOLD, - // the annotation is considered a duplicate. - return distance < PROXIMITY_THRESHOLD; - }, - ); - - if (!isDuplicate) { - deduplicatedTextAnnotations.push(annotation); - } - } - } - - // **8:** Restore the original DOM after mutations - await this.stagehandPage.page.evaluate( - ({ dom, xp }) => window.restoreDOM(dom, xp), - { dom: originalDOM, xp: targetXpath }, - ); - - // **9:** Format the deduplicated annotations into a text representation - const formattedText = formatText( - deduplicatedTextAnnotations, - containerWidth, - ); - - // **10:** Pass the formatted text to an LLM for extraction according to the given instruction and schema - const extractionResponse = await extract({ - instruction, - previouslyExtractedContent: content, - domElements: formattedText, - schema, - chunksSeen: 1, - chunksTotal: 1, - llmClient, - requestId, - userProvidedInstructions: this.userProvidedInstructions, - logger: this.logger, - logInferenceToFile: this.stagehand.logInferenceToFile, - }); - - const { - metadata: { completed }, - prompt_tokens: promptTokens, - completion_tokens: completionTokens, - inference_time_ms: inferenceTimeMs, - ...output - } = extractionResponse; - - this.stagehand.updateMetrics( - StagehandFunctionName.EXTRACT, - promptTokens, - completionTokens, - inferenceTimeMs, - ); - - // **11:** Handle the extraction response and log the results - this.logger({ - category: "extraction", - message: "received extraction response", - auxiliary: { - extraction_response: { - value: JSON.stringify(extractionResponse), - type: "object", - }, - }, - }); - - if (completed) { - this.logger({ - category: "extraction", - message: "extraction completed successfully", - level: 1, - auxiliary: { - extraction_response: { - value: JSON.stringify(extractionResponse), - type: "object", - }, - }, - }); - } else { - this.logger({ - category: "extraction", - message: "extraction incomplete after processing all data", - level: 1, - auxiliary: { - extraction_response: { - value: JSON.stringify(extractionResponse), - type: "object", - }, - }, - }); - } - return output; - } - - private async domExtract({ - instruction, - schema, - content = {}, - llmClient, - requestId, - domSettleTimeoutMs, - }: { - instruction: string; - schema: T; - content?: z.infer; - llmClient: LLMClient; - requestId?: string; - domSettleTimeoutMs?: number; - }): Promise> { - this.logger({ - category: "extraction", - message: "starting extraction using a11y tree", - level: 1, - auxiliary: { - instruction: { - value: instruction, - type: "string", - }, - }, - }); - - await this.stagehandPage._waitForSettledDom(domSettleTimeoutMs); - const tree = await getAccessibilityTree(this.stagehandPage, this.logger); - this.logger({ - category: "extraction", - message: "Getting accessibility tree data", - level: 1, - }); - const outputString = tree.simplified; - const idToUrlMapping = tree.idToUrl; - - // Transform user defined schema to replace string().url() with .number() - const [transformedSchema, urlFieldPaths] = - transformUrlStringsToNumericIds(schema); - - // call extract inference with transformed schema - const extractionResponse = await extract({ - instruction, - previouslyExtractedContent: content, - domElements: outputString, - schema: transformedSchema, - chunksSeen: 1, - chunksTotal: 1, - llmClient, - requestId, - userProvidedInstructions: this.userProvidedInstructions, - logger: this.logger, - logInferenceToFile: this.stagehand.logInferenceToFile, - }); - - const { - metadata: { completed }, - prompt_tokens: promptTokens, - completion_tokens: completionTokens, - inference_time_ms: inferenceTimeMs, - ...output - } = extractionResponse; - - this.stagehand.updateMetrics( - StagehandFunctionName.EXTRACT, - promptTokens, - completionTokens, - inferenceTimeMs, - ); - - this.logger({ - category: "extraction", - message: "received extraction response", - auxiliary: { - extraction_response: { - value: JSON.stringify(extractionResponse), - type: "object", - }, - }, - }); - - if (completed) { - this.logger({ - category: "extraction", - message: "extraction completed successfully", - level: 1, - auxiliary: { - extraction_response: { - value: JSON.stringify(extractionResponse), - type: "object", - }, - }, - }); - } else { - this.logger({ - category: "extraction", - message: "extraction incomplete after processing all data", - level: 1, - auxiliary: { - extraction_response: { - value: JSON.stringify(extractionResponse), - type: "object", - }, - }, - }); - } - - // revert to original schema and populate with URLs - for (const { segments } of urlFieldPaths) { - injectUrls(output, segments, idToUrlMapping); - } - - return output as z.infer; - } - /** - * Get the width, height, and offsets of either the entire page or a specific element. - * (Matches your existing getTargetDimensions logic, just adapted to accept a string | undefined.) - */ - private async getTargetDimensions(targetXpath?: string): Promise<{ - width: number; - height: number; - offsetLeft: number; - offsetTop: number; - }> { - // If targetXpath is undefined, get entire page dimensions - if (!targetXpath) { - const { innerWidth, innerHeight } = await this.stagehand.page.evaluate( - () => ({ - innerWidth: window.innerWidth, - innerHeight: window.innerHeight, - }), - ); - return { - width: innerWidth, - height: innerHeight, - offsetLeft: 0, - offsetTop: 0, - }; - } - - // If targetXpath is present, get element-specific dimensions - const { elemWidth, elemHeight, offsetLeft, offsetTop } = - await this.stagehand.page.evaluate((xp) => { - const el = window.getNodeFromXpath(xp) as HTMLElement | null; - - if (!el) { - return { - elemWidth: window.innerWidth, - elemHeight: window.innerHeight, - offsetLeft: 0, - offsetTop: 0, - }; - } - - const rect = el.getBoundingClientRect(); - return { - elemWidth: rect.width, - elemHeight: rect.height, - offsetLeft: rect.left, - offsetTop: rect.top, - }; - }, targetXpath); - - return { - width: elemWidth, - height: elemHeight, - offsetLeft, - offsetTop, - }; - } - - /** - * Collects the bounding boxes for each word inside each of the candidate element in selectorMap, - * adjusting for container offsets, and producing an array of TextAnnotations. - */ - private async collectAllAnnotations( - selectorMap: Record, - containerWidth: number, - containerHeight: number, - offsetLeft: number, - offsetTop: number, - ): Promise { - const allAnnotations: TextAnnotation[] = []; - - // Loop over the candidate XPaths in the selector map - for (const xpaths of Object.values(selectorMap)) { - const xpath = xpaths[0]; - - // Evaluate in the browser to get bounding boxes - const boundingBoxes: Array<{ - text: string; - left: number; - top: number; - width: number; - height: number; - }> = await this.stagehandPage.page.evaluate( - (xp) => window.getElementBoundingBoxes(xp), - xpath, - ); - - for (const box of boundingBoxes) { - // 1. Subtract container offsets to get local coordinates - const localLeft = box.left - offsetLeft; - const localTop = box.top - offsetTop; - - // 2. bottom_left is local x, plus local y + height - // so the baseline is at the bottom edge of the box - const bottom_left = { x: localLeft, y: localTop + box.height }; - - // 3. Normalize by dividing local positions by container width/height - const bottom_left_normalized = { - x: localLeft / containerWidth, - y: (localTop + box.height) / containerHeight, - }; - - if (box.text.trim().length > 0) { - allAnnotations.push({ - text: box.text, - bottom_left, - bottom_left_normalized, - width: box.width, - height: box.height, - }); - } - } - } - - return allAnnotations; - } - - /** - * Deduplicate text annotations by grouping them by text, then removing duplicates - * within a certain proximity threshold. - */ - private deduplicateAnnotations( - annotations: TextAnnotation[], - ): TextAnnotation[] { - const annotationsGroupedByText = new Map(); - const deduplicated: TextAnnotation[] = []; - - for (const annotation of annotations) { - if (!annotationsGroupedByText.has(annotation.text)) { - annotationsGroupedByText.set(annotation.text, []); - } - annotationsGroupedByText.get(annotation.text)!.push(annotation); - } - - for (const [text, group] of annotationsGroupedByText.entries()) { - for (const annotation of group) { - const isDuplicate = deduplicated.some((existing) => { - if (existing.text !== text) return false; - - const dx = existing.bottom_left.x - annotation.bottom_left.x; - const dy = existing.bottom_left.y - annotation.bottom_left.y; - const distance = Math.hypot(dx, dy); - return distance < PROXIMITY_THRESHOLD; - }); - - if (!isDuplicate) { - deduplicated.push(annotation); - } - } - } - - return deduplicated; - } -} - -/** - * Scans the provided Zod schema for any `z.string().url()` fields and - * replaces them with `z.number()`. - * - * @param schema - The Zod object schema to transform. - * @returns A tuple containing: - * 1. The transformed schema (or the original schema if no changes were needed). - * 2. An array of {@link ZodPathSegments} objects representing all the replaced URL fields, - * with each path segment showing where in the schema the replacement occurred. - */ -export function transformUrlStringsToNumericIds< - T extends z.ZodObject, ->(schema: T): [T, ZodPathSegments[]] { - const shape = schema._def.shape(); - const newShape: Record = {}; - const urlPaths: ZodPathSegments[] = []; - let changed = false; - - for (const [key, value] of Object.entries(shape)) { - const [childTransformed, childPaths] = transformSchema(value, [key]); - newShape[key] = childTransformed; - if (childTransformed !== value) { - changed = true; - } - if (childPaths.length > 0) { - childPaths.forEach((cp) => { - urlPaths.push({ segments: [key, ...cp.segments] }); - }); - } - } - - const finalSchema = changed ? z.object(newShape) : schema; - return [finalSchema as T, urlPaths]; -} - -/** - * Recursively traverses a given Zod schema, scanning for any fields of type `z.string().url()`. - * For each such field, it replaces the `z.string().url()` with `z.number()`. - * - * This function is used internally by higher-level utilities (e.g., transforming entire object schemas) - * and handles nested objects, arrays, unions, intersections, optionals. - * - * @param schema - The Zod schema to transform. - * @param currentPath - An array of string/number keys representing the current schema path (used internally for recursion). - * @returns A two-element tuple: - * 1. The updated Zod schema, with any `.url()` fields replaced by `z.number()`. - * 2. An array of {@link ZodPathSegments} objects representing each replaced field, including the path segments. - */ -export function transformSchema( - schema: ZodTypeAny, - currentPath: Array, -): [ZodTypeAny, ZodPathSegments[]] { - // 1) If it's a string with .url(), convert to z.number() - if (schema instanceof z.ZodString) { - const hasUrlCheck = - schema._def.checks?.some((check) => check.kind === "url") ?? false; - if (hasUrlCheck) { - return [ - z.number().describe("ID of element that points to a URL"), - [{ segments: [] }], - ]; - } - return [schema, []]; - } - - // 2) If it's an object, transform each field - if (schema instanceof z.ZodObject) { - // The shape is a raw object containing fields keyed by string (no symbols): - const shape = schema._def.shape() as Record; - const newShape: Record = {}; - const urlPaths: ZodPathSegments[] = []; - let changed = false; - - const shapeKeys = Object.keys(shape); - - for (const key of shapeKeys) { - const child = shape[key]; - const [transformedChild, childPaths] = transformSchema(child, [ - ...currentPath, - key, - ]); - - if (transformedChild !== child) { - changed = true; - } - newShape[key] = transformedChild; - - if (childPaths.length > 0) { - for (const cp of childPaths) { - urlPaths.push({ segments: [key, ...cp.segments] }); - } - } - } - - if (changed) { - return [z.object(newShape), urlPaths]; - } - return [schema, urlPaths]; - } - - // 3) If it's an array, transform its item type - if (schema instanceof z.ZodArray) { - const itemType = schema._def.type as ZodTypeAny; - const [transformedItem, childPaths] = transformSchema(itemType, [ - ...currentPath, - "*", - ]); - const changed = transformedItem !== itemType; - const arrayPaths: ZodPathSegments[] = childPaths.map((cp) => ({ - segments: ["*", ...cp.segments], - })); - - if (changed) { - return [z.array(transformedItem), arrayPaths]; - } - return [schema, arrayPaths]; - } - - // 4) If it's a union, transform each option - if (schema instanceof z.ZodUnion) { - // Cast the union’s options to an array of ZodTypeAny - const unionOptions = schema._def.options as ZodTypeAny[]; - const newOptions: ZodTypeAny[] = []; - let changed = false; - let allPaths: ZodPathSegments[] = []; - - unionOptions.forEach((option: ZodTypeAny, idx: number) => { - const [newOption, childPaths] = transformSchema(option, [ - ...currentPath, - `union_${idx}`, - ]); - if (newOption !== option) { - changed = true; - } - newOptions.push(newOption); - allPaths = [...allPaths, ...childPaths]; - }); - - if (changed) { - // We assume at least two options remain: - return [ - z.union(newOptions as [ZodTypeAny, ZodTypeAny, ...ZodTypeAny[]]), - allPaths, - ]; - } - return [schema, allPaths]; - } - - // 5) If it's an intersection, transform left and right - if (schema instanceof z.ZodIntersection) { - const leftType = schema._def.left as ZodTypeAny; - const rightType = schema._def.right as ZodTypeAny; - - const [left, leftPaths] = transformSchema(leftType, [ - ...currentPath, - "intersection_left", - ]); - const [right, rightPaths] = transformSchema(rightType, [ - ...currentPath, - "intersection_right", - ]); - const changed = left !== leftType || right !== rightType; - const allPaths = [...leftPaths, ...rightPaths]; - if (changed) { - return [z.intersection(left, right), allPaths]; - } - return [schema, allPaths]; - } - - // 6) If it's optional, transform inner - if (schema instanceof z.ZodOptional) { - const innerType = schema._def.innerType as ZodTypeAny; - const [inner, innerPaths] = transformSchema(innerType, currentPath); - if (inner !== innerType) { - return [z.optional(inner), innerPaths]; - } - return [schema, innerPaths]; - } - - // 7) If it's nullable, transform inner - if (schema instanceof z.ZodNullable) { - const innerType = schema._def.innerType as ZodTypeAny; - const [inner, innerPaths] = transformSchema(innerType, currentPath); - if (inner !== innerType) { - return [z.nullable(inner), innerPaths]; - } - return [schema, innerPaths]; - } - - // 8) If it's an effect, transform base schema - if (schema instanceof z.ZodEffects) { - const baseSchema = schema._def.schema as ZodTypeAny; - const [newBaseSchema, basePaths] = transformSchema(baseSchema, currentPath); - if (newBaseSchema !== baseSchema) { - return [z.effect(newBaseSchema, schema._def.effect), basePaths]; - } - return [schema, basePaths]; - } - - // 9) If none of the above, return as-is - return [schema, []]; -} - -/** - * Once we get the final extracted object that has numeric IDs in place of URLs, - * use `injectUrls` to walk the object and replace numeric IDs - * with the real URL strings from idToUrlMapping. The `path` may include `*` - * for array indices (indicating "all items in the array"). - */ -export function injectUrls( - obj: unknown, - path: Array, - idToUrlMapping: Record, -): void { - if (path.length === 0) return; - const [key, ...rest] = path; - - if (key === "*") { - if (Array.isArray(obj)) { - for (const item of obj) { - injectUrls(item, rest, idToUrlMapping); - } - } - return; - } - - if (obj && typeof obj === "object") { - const record = obj as Record; - if (path.length === 1) { - const fieldValue = record[key]; - if (typeof fieldValue === "number") { - const mappedUrl = idToUrlMapping[String(fieldValue)]; - record[key] = mappedUrl ?? ``; - } - } else { - injectUrls(record[key], rest, idToUrlMapping); - } - } -} diff --git a/lib/handlers/handlerUtils/actHandlerUtils.ts b/lib/handlers/handlerUtils/actHandlerUtils.ts deleted file mode 100644 index 3cc41acc0..000000000 --- a/lib/handlers/handlerUtils/actHandlerUtils.ts +++ /dev/null @@ -1,570 +0,0 @@ -import { Page, Locator, errors as PlaywrightErrors } from "@playwright/test"; -import { PlaywrightCommandException } from "../../../types/playwright"; -import { StagehandPage } from "../../StagehandPage"; -import { getNodeFromXpath } from "@/lib/dom/utils"; -import { Logger } from "../../../types/log"; -import { MethodHandlerContext } from "@/types/act"; - -/** - * A mapping of playwright methods that may be chosen by the LLM to their - * implementation. - */ -export const methodHandlerMap: Record< - string, - (ctx: MethodHandlerContext) => Promise -> = { - scrollIntoView: scrollElementIntoView, - scrollTo: scrollElementToPercentage, - scroll: scrollElementToPercentage, - "mouse.wheel": scrollElementToPercentage, - fill: fillOrType, - type: fillOrType, - press: pressKey, - click: clickElement, - nextChunk: scrollToNextChunk, - prevChunk: scrollToPreviousChunk, -}; - -export async function scrollToNextChunk(ctx: MethodHandlerContext) { - const { stagehandPage, xpath, logger } = ctx; - - logger({ - category: "action", - message: "scrolling to next chunk", - level: 2, - auxiliary: { - xpath: { value: xpath, type: "string" }, - }, - }); - - try { - await stagehandPage.page.evaluate( - ({ xpath }) => { - const elementNode = getNodeFromXpath(xpath); - if (!elementNode || elementNode.nodeType !== Node.ELEMENT_NODE) { - console.warn(`Could not locate element to scroll by its height.`); - return Promise.resolve(); - } - - const element = elementNode as HTMLElement; - const tagName = element.tagName.toLowerCase(); - let height: number; - - if (tagName === "html" || tagName === "body") { - height = window.visualViewport.height; - window.scrollBy({ - top: height, - left: 0, - behavior: "smooth", - }); - - const scrollingEl = - document.scrollingElement || document.documentElement; - return window.waitForElementScrollEnd(scrollingEl as HTMLElement); - } else { - height = element.getBoundingClientRect().height; - element.scrollBy({ - top: height, - left: 0, - behavior: "smooth", - }); - - return window.waitForElementScrollEnd(element); - } - }, - { xpath }, - ); - } catch (e) { - logger({ - category: "action", - message: "error scrolling to next chunk", - level: 1, - auxiliary: { - error: { value: e.message, type: "string" }, - trace: { value: e.stack, type: "string" }, - xpath: { value: xpath, type: "string" }, - }, - }); - throw new PlaywrightCommandException(e.message); - } -} - -export async function scrollToPreviousChunk(ctx: MethodHandlerContext) { - const { stagehandPage, xpath, logger } = ctx; - - logger({ - category: "action", - message: "scrolling to previous chunk", - level: 2, - auxiliary: { - xpath: { value: xpath, type: "string" }, - }, - }); - - try { - await stagehandPage.page.evaluate( - ({ xpath }) => { - const elementNode = getNodeFromXpath(xpath); - if (!elementNode || elementNode.nodeType !== Node.ELEMENT_NODE) { - console.warn(`Could not locate element to scroll by its height.`); - return Promise.resolve(); - } - - const element = elementNode as HTMLElement; - const tagName = element.tagName.toLowerCase(); - let height: number; - - if (tagName === "html" || tagName === "body") { - height = window.visualViewport.height; - window.scrollBy({ - top: -height, - left: 0, - behavior: "smooth", - }); - - const scrollingEl = - document.scrollingElement || document.documentElement; - return window.waitForElementScrollEnd(scrollingEl as HTMLElement); - } else { - height = element.getBoundingClientRect().height; - element.scrollBy({ - top: -height, - left: 0, - behavior: "smooth", - }); - return window.waitForElementScrollEnd(element); - } - }, - { xpath }, - ); - } catch (e) { - logger({ - category: "action", - message: "error scrolling to previous chunk", - level: 1, - auxiliary: { - error: { value: e.message, type: "string" }, - trace: { value: e.stack, type: "string" }, - xpath: { value: xpath, type: "string" }, - }, - }); - throw new PlaywrightCommandException(e.message); - } -} - -export async function scrollElementIntoView(ctx: MethodHandlerContext) { - const { locator, xpath, logger } = ctx; - - logger({ - category: "action", - message: "scrolling element into view", - level: 2, - auxiliary: { - xpath: { value: xpath, type: "string" }, - }, - }); - - try { - await locator.evaluate((element: HTMLElement) => { - element.scrollIntoView({ behavior: "smooth", block: "center" }); - }); - } catch (e) { - logger({ - category: "action", - message: "error scrolling element into view", - level: 1, - auxiliary: { - error: { value: e.message, type: "string" }, - trace: { value: e.stack, type: "string" }, - xpath: { value: xpath, type: "string" }, - }, - }); - throw new PlaywrightCommandException(e.message); - } -} - -export async function scrollElementToPercentage(ctx: MethodHandlerContext) { - const { args, stagehandPage, xpath, logger } = ctx; - - logger({ - category: "action", - message: "scrolling element vertically to specified percentage", - level: 2, - auxiliary: { - xpath: { value: xpath, type: "string" }, - coordinate: { value: JSON.stringify(args), type: "string" }, - }, - }); - - try { - const [yArg = "0%"] = args as string[]; - - await stagehandPage.page.evaluate( - ({ xpath, yArg }) => { - function parsePercent(val: string): number { - const cleaned = val.trim().replace("%", ""); - const num = parseFloat(cleaned); - return Number.isNaN(num) ? 0 : Math.max(0, Math.min(num, 100)); - } - - const elementNode = getNodeFromXpath(xpath); - if (!elementNode || elementNode.nodeType !== Node.ELEMENT_NODE) { - console.warn(`Could not locate element to scroll on.`); - return; - } - - const element = elementNode as HTMLElement; - const yPct = parsePercent(yArg); - - if (element.tagName.toLowerCase() === "html") { - const scrollHeight = document.body.scrollHeight; - const viewportHeight = window.innerHeight; - const scrollTop = (scrollHeight - viewportHeight) * (yPct / 100); - window.scrollTo({ - top: scrollTop, - left: window.scrollX, - behavior: "smooth", - }); - } else { - const scrollHeight = element.scrollHeight; - const clientHeight = element.clientHeight; - const scrollTop = (scrollHeight - clientHeight) * (yPct / 100); - element.scrollTo({ - top: scrollTop, - left: element.scrollLeft, - behavior: "smooth", - }); - } - }, - { xpath, yArg }, - ); - } catch (e) { - logger({ - category: "action", - message: "error scrolling element vertically to percentage", - level: 1, - auxiliary: { - error: { value: e.message, type: "string" }, - trace: { value: e.stack, type: "string" }, - xpath: { value: xpath, type: "string" }, - args: { value: JSON.stringify(args), type: "object" }, - }, - }); - throw new PlaywrightCommandException(e.message); - } -} - -export async function fillOrType(ctx: MethodHandlerContext) { - const { locator, xpath, args, logger } = ctx; - - try { - await locator.fill(""); - await locator.click(); - - const text = args[0]?.toString() || ""; - for (const char of text) { - await locator.page().keyboard.type(char, { - delay: Math.random() * 50 + 25, - }); - } - } catch (e) { - logger({ - category: "action", - message: "error filling element", - level: 1, - auxiliary: { - error: { value: e.message, type: "string" }, - trace: { value: e.stack, type: "string" }, - xpath: { value: xpath, type: "string" }, - }, - }); - throw new PlaywrightCommandException(e.message); - } -} - -export async function pressKey(ctx: MethodHandlerContext) { - const { - locator, - xpath, - args, - logger, - stagehandPage, - initialUrl, - domSettleTimeoutMs, - } = ctx; - try { - const key = args[0]?.toString() ?? ""; - await locator.page().keyboard.press(key); - - await handlePossiblePageNavigation( - "press", - xpath, - initialUrl, - stagehandPage, - logger, - domSettleTimeoutMs, - ); - } catch (e) { - logger({ - category: "action", - message: "error pressing key", - level: 1, - auxiliary: { - error: { value: e.message, type: "string" }, - trace: { value: e.stack, type: "string" }, - key: { value: args[0]?.toString() ?? "unknown", type: "string" }, - }, - }); - throw new PlaywrightCommandException(e.message); - } -} - -export async function clickElement(ctx: MethodHandlerContext) { - const { - locator, - xpath, - args, - logger, - stagehandPage, - initialUrl, - domSettleTimeoutMs, - } = ctx; - - logger({ - category: "action", - message: "page URL before click", - level: 2, - auxiliary: { - url: { - value: stagehandPage.page.url(), - type: "string", - }, - }, - }); - - try { - // If it's a radio input, try to click its label - const isRadio = await locator.evaluate((el) => { - return el instanceof HTMLInputElement && el.type === "radio"; - }); - - // Extract the click options (if any) from args[0] - const clickArg = (args[0] ?? {}) as Record; - - // Decide which locator we actually want to click (for radio inputs, prefer label if present) - let finalLocator = locator; - if (isRadio) { - const inputId = await locator.evaluate( - (el) => (el as HTMLInputElement).id, - ); - let labelLocator = null; - - if (inputId) { - labelLocator = stagehandPage.page.locator(`label[for="${inputId}"]`); - } - if (!labelLocator || (await labelLocator.count()) < 1) { - // Check ancestor