feat(mcp): add agent-focused web tools

CHANGELOG.md

@@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Renamed internal byte/character limits to `STREAM_THRESHOLD_BYTES` and `MCP_HTML_MAX_CHARS`, and namespaced the Camoufox font-retention build argument as `CAMOUFOX_KEEP_SPOOFED_OS_FONTS`.
 
 ### Added
+- Expanded the optional MCP server with purpose-specific `read`, `scrape`, `screenshot`, and `inspect` tools. AI clients can now receive readability-extracted Markdown/text, native image content, richer scrape metadata, or bounded browser diagnostics without exposing cookies, sessions, request headers, proxy credentials, or captured API response bodies. The original `scrape_url` remains as a compatibility alias.
 - Optional `MITM_ALWAYS_SCRAPE=true` mode skips the forward proxy's direct Tier 0 probe and routes ordinary HTTP requests immediately into the existing scraper ladder for sites where the probe itself triggers a temporary ban (#93). WebSocket relays remain direct; the mode is off by default and documented as unsuitable for general media/download traffic because it bypasses Tier 0 streaming.
 - **DataDome support.** Detect Device Check, slider CAPTCHA and `t=bv` hard blocks from challenge markers and `x-dd-b`. Device Check uses a dedicated waiter and an optional headful Xvfb pool; the slider is reported as `datadome-captcha-required`. Enable startup-warmed capacity with `BROWSER_HEADFUL_POOL_SIZE=1` (off by default).
 - **AWS WAF Challenge support.** Detect the documented `202` Challenge and `405` CAPTCHA responses from their `x-amzn-waf-action` header, with a conservative two-marker HTML fallback. Silent challenges use a dedicated browser waiter for the domain-matching `aws-waf-token`; interactive CAPTCHA is surfaced as `aws-waf-captcha-required` for a future solver.

README.md

@@ -118,11 +118,12 @@ curl -X POST http://localhost:8191/scrape \
   -d '{"url":"https://nowsecure.nl","maxTimeout":60000}'
 ```
 
-### MCP scraping tool (`/mcp`)
+### MCP tools (`/mcp`)
 
-Set `MCP_ENABLED=true` to expose TRAWL's client-independent Streamable HTTP
-`scrape_url` tool to any MCP-compatible AI application or agent. It loads known
-public URLs; it does not provide web search or ranking. See the
+Set `MCP_ENABLED=true` to expose TRAWL's client-independent Streamable HTTP tools
+for readable content, HTML, screenshots and browser diagnostics to any
+MCP-compatible AI application or agent. They load known public URLs; TRAWL does not
+provide web search or ranking. See the
 [MCP integration guide](./apps/docs/integrations/mcp.md).
 
 ### Connect Prowlarr / Jackett

apps/api/package.json

@@ -9,17 +9,21 @@
   },
   "dependencies": {
     "@modelcontextprotocol/sdk": "^1.30.0",
+    "@mozilla/readability": "^0.6.0",
     "@trawl/browser": "workspace:*",
     "@trawl/tiers": "workspace:*",
     "@trawl/types": "workspace:*",
     "elysia": "^1.4.29",
+    "linkedom": "^0.18.13",
     "memoirist": "1.2.2",
     "node-forge": "^1.4.0",
+    "turndown": "^7.2.4",
     "zod": "^4.4.3"
   },
   "devDependencies": {
-    "typescript": "^7.0.2",
     "@types/bun": "^1.4.0",
-    "@types/node-forge": "^1.3.14"
+    "@types/node-forge": "^1.3.14",
+    "@types/turndown": "^5.0.6",
+    "typescript": "^7.0.2"
   }
 }

apps/api/src/app.test.ts

@@ -24,7 +24,7 @@ describe("MCP opt-in registration", () => {
     try {
       await client.connect(new StreamableHTTPClientTransport(new URL(`http://127.0.0.1:${server.port}/mcp`)))
       const tools = await client.listTools()
-      expect(tools.tools.map((tool) => tool.name)).toEqual(["scrape_url"])
+      expect(tools.tools.map((tool) => tool.name)).toEqual(["scrape", "scrape_url", "read", "screenshot", "inspect"])
     } finally {
       await client.close()
       server.stop(true)

apps/api/src/routes/mcp.test.ts

@@ -28,7 +28,7 @@ function rpc(method: string, params?: unknown, id = 1): Request {
 }
 
 describe("MCP route", () => {
-  test("initializes and lists only scrape_url", async () => {
+  test("initializes and lists the focused tool set with the compatibility alias", async () => {
     const app = mcpRoute({ poolReady: () => true, runScrape: async () => baseResult })
     const initialized = await app.handle(
       rpc("initialize", {
@@ -43,7 +43,13 @@ describe("MCP route", () => {
     const listed = await app.handle(rpc("tools/list"))
     expect(listed.status).toBe(200)
     const body = await listed.json()
-    expect(body.result.tools.map((tool: { name: string }) => tool.name)).toEqual(["scrape_url"])
+    expect(body.result.tools.map((tool: { name: string }) => tool.name)).toEqual([
+      "scrape",
+      "scrape_url",
+      "read",
+      "screenshot",
+      "inspect",
+    ])
   })
 
   test("exposes the Streamable HTTP GET channel", async () => {
@@ -75,12 +81,92 @@ describe("MCP route", () => {
       contentType: "text/html; charset=utf-8",
       totalMs: 12,
       truncated: true,
+      sessionCached: true,
+      timings: baseResult.timings,
     })
     expect(body.result.content[0].text.length).toBe(MCP_HTML_MAX_CHARS)
     expect(JSON.stringify(body)).not.toContain('cookie"')
     expect(JSON.stringify(body)).not.toContain("secret-agent")
   })
 
+  test("extracts readable markdown with metadata and a caller-controlled limit", async () => {
+    const app = mcpRoute({
+      poolReady: () => true,
+      runScrape: async () => ({
+        ...baseResult,
+        html: `<!doctype html><html lang="en"><head><title>Ignored shell title</title></head><body><article><h1>Useful title</h1><p>${"Readable content ".repeat(40)}</p></article></body></html>`,
+      }),
+    })
+    const response = await app.handle(
+      rpc("tools/call", {
+        name: "read",
+        arguments: { url: "https://1.1.1.1/article", format: "markdown", maxCharacters: 80 },
+      }),
+    )
+    const result = (await response.json()).result
+    expect(result.isError).toBeUndefined()
+    expect(result.content[0].text).toContain("Useful title")
+    expect(result.content[0].text.length).toBe(80)
+    expect(result.structuredContent).toMatchObject({ format: "markdown", characters: 80, truncated: true })
+  })
+
+  test("returns screenshots as MCP image content and forces a browser tier", async () => {
+    let received: unknown
+    const app = mcpRoute({
+      poolReady: () => true,
+      runScrape: async (input) => {
+        received = input
+        return { ...baseResult, screenshot: "aGVsbG8=" }
+      },
+    })
+    const response = await app.handle(
+      rpc("tools/call", { name: "screenshot", arguments: { url: "https://1.1.1.1", maxTier: 3 } }),
+    )
+    const result = (await response.json()).result
+    expect(received).toEqual({ url: "https://1.1.1.1", maxTier: 3, skipHttp: true, screenshot: true })
+    expect(result.content[0]).toEqual({ type: "image", data: "aGVsbG8=", mimeType: "image/jpeg" })
+    expect(result.structuredContent.mimeType).toBe("image/jpeg")
+  })
+
+  test("returns browser diagnostics while redacting URL credentials and query strings", async () => {
+    let received: unknown
+    const app = mcpRoute({
+      poolReady: () => true,
+      runScrape: async (input) => {
+        received = input
+        return {
+          ...baseResult,
+          consoleLogs: [{ level: "SEVERE", message: "boom", timestamp: 1, source: "error" }],
+          networkLogs: [
+            {
+              name: "https://user:secret@example.com/api?token=secret#part",
+              entryType: "resource",
+              startTime: 1,
+              duration: 2,
+              initiatorType: "fetch",
+              transferSize: 3,
+              encodedBodySize: 2,
+              decodedBodySize: null,
+            },
+          ],
+          redirectChain: ["https://example.com/start?token=secret", "https://example.com/final#private"],
+        }
+      },
+    })
+    const response = await app.handle(rpc("tools/call", { name: "inspect", arguments: { url: "https://1.1.1.1" } }))
+    const result = (await response.json()).result
+    expect(received).toEqual({
+      url: "https://1.1.1.1",
+      skipHttp: true,
+      consoleLogs: true,
+      networkLogs: true,
+      redirectChain: true,
+    })
+    expect(result.structuredContent.networkLogs[0].name).toBe("https://example.com/api")
+    expect(result.structuredContent.redirectChain).toEqual(["https://example.com/start", "https://example.com/final"])
+    expect(JSON.stringify(result)).not.toContain("secret")
+  })
+
   test("rejects extra arguments, private targets, and disallowed origins", async () => {
     const app = mcpRoute({
       allowedOrigins: ["https://chat.example"],