From fdd8a336a368f9fc184e0be746a3ddb50ee6568c Mon Sep 17 00:00:00 2001 From: germondai Date: Fri, 4 Sep 2026 02:41:29 +0200 Subject: [PATCH 1/4] chore(api): add readable content dependencies --- apps/api/package.json | 8 +++-- bun.lock | 68 ++++++++++++++++++++++++++++++++++++++----- 2 files changed, 66 insertions(+), 10 deletions(-) diff --git a/apps/api/package.json b/apps/api/package.json index d68978a..0faaed1 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -9,17 +9,21 @@ }, "dependencies": { "@modelcontextprotocol/sdk": "^1.30.0", + "@mozilla/readability": "^0.6.0", "@trawl/browser": "workspace:*", "@trawl/tiers": "workspace:*", "@trawl/types": "workspace:*", "elysia": "^1.4.29", + "linkedom": "^0.18.13", "memoirist": "1.2.2", "node-forge": "^1.4.0", + "turndown": "^7.2.4", "zod": "^4.4.3" }, "devDependencies": { - "typescript": "^7.0.2", "@types/bun": "^1.4.0", - "@types/node-forge": "^1.3.14" + "@types/node-forge": "^1.3.14", + "@types/turndown": "^5.0.6", + "typescript": "^7.0.2" } } diff --git a/bun.lock b/bun.lock index 658fbc3..7d2f2b1 100644 --- a/bun.lock +++ b/bun.lock @@ -13,17 +13,21 @@ "version": "1.4.2", "dependencies": { "@modelcontextprotocol/sdk": "^1.30.0", + "@mozilla/readability": "^0.6.0", "@trawl/browser": "workspace:*", "@trawl/tiers": "workspace:*", "@trawl/types": "workspace:*", "elysia": "^1.4.29", + "linkedom": "^0.18.13", "memoirist": "1.2.2", "node-forge": "^1.4.0", + "turndown": "^7.2.4", "zod": "^4.4.3", }, "devDependencies": { "@types/bun": "^1.4.0", "@types/node-forge": "^1.3.14", + "@types/turndown": "^5.0.6", "typescript": "^7.0.2", }, }, @@ -357,8 +361,12 @@ "@mapbox/node-pre-gyp": ["@mapbox/node-pre-gyp@2.0.3", "", { "dependencies": { "consola": "^3.2.3", "detect-libc": "^2.0.0", "https-proxy-agent": "^7.0.5", "node-fetch": "^2.6.7", "nopt": "^8.0.0", "semver": "^7.5.3", "tar": "^7.4.0" }, "bin": { "node-pre-gyp": "bin/node-pre-gyp" } }, "sha512-uwPAhccfFJlsfCxMYTwOdVfOz3xqyj8xYL3zJj8f0pb30tLohnnFPhLuqp4/qoEz8sNxe4SESZedcBojRefIzg=="], + "@mixmark-io/domino": ["@mixmark-io/domino@2.2.0", "", {}, "sha512-Y28PR25bHXUg88kCV7nivXrP2Nj2RueZ3/l/jdx6J9f8J4nsEGcgX0Qe6lt7Pa+J79+kPiJU3LguR6O/6zrLOw=="], + "@modelcontextprotocol/sdk": ["@modelcontextprotocol/sdk@1.30.0", "", { "dependencies": { "@hono/node-server": "^1.19.9 || ^2.0.5", "ajv": "^8.17.1", "ajv-formats": "^3.0.1", "content-type": "^1.0.5", "cors": "^2.8.5", "cross-spawn": "^7.0.5", "eventsource": "^3.0.2", "eventsource-parser": "^3.0.0", "express": "^5.2.1", "express-rate-limit": "^8.2.1", "hono": "^4.11.4", "jose": "^6.1.3", "json-schema-typed": "^8.0.2", "pkce-challenge": "^5.0.0", "raw-body": "^3.0.0", "zod": "^3.25 || ^4.0", "zod-to-json-schema": "^3.25.1" }, "peerDependencies": { "@cfworker/json-schema": "^4.1.1" }, "optionalPeers": ["@cfworker/json-schema"] }, "sha512-xKd8OIzlqNzcqcNumGAa6g+PW2kjD5vrpcKOnfldAUPP3j7lnqMPwlTXQm8gF+UwH72z0lqaRbjr9hqGz0eITA=="], + "@mozilla/readability": ["@mozilla/readability@0.6.0", "", {}, "sha512-juG5VWh4qAivzTAeMzvY9xs9HY5rAcr2E4I7tiSSCokRFi7XIZCAu92ZkSTsIj1OPceCifL3cpfteP3pDT9/QQ=="], + "@napi-rs/lzma-linux-x64-gnu": ["@napi-rs/lzma-linux-x64-gnu@1.5.1", "", { "os": "linux", "cpu": "x64" }, "sha512-oTXEIha4SsuXdTA4Iyskj0kpdx2yVXdhd75c2v3xGrHFfVMsbhTPZU/nMPL4sWKo4pBHm3aucLaqGlF696dTyQ=="], "@napi-rs/wasm-runtime": ["@napi-rs/wasm-runtime@1.2.2", "", { "dependencies": { "@tybys/wasm-util": "^0.10.3" }, "peerDependencies": { "@emnapi/core": "^1.7.1 || ^2.0.0-alpha.3", "@emnapi/runtime": "^1.7.1 || ^2.0.0-alpha.3" } }, "sha512-JfB4kuJQjaoHuCTseIINHtHWeJnvgEcxjwA5t/Y00ZgaOO1Crz3fjT/p8kT28zA/Caz7oiUMn3d6H2yOVCVwuw=="], @@ -619,6 +627,8 @@ "@types/resolve": ["@types/resolve@1.20.2", "", {}, "sha512-60BCwRFOZCQhDncwQdxxeOEEkbc5dIMccYLwbxsS4TUNeVECQ/pBJ0j09mrHOl/JJvpRPGwO9SvE4nR2Nb/a4Q=="], + "@types/turndown": ["@types/turndown@5.0.6", "", {}, "sha512-ru00MoyeeouE5BX4gRL+6m/BsDfbRayOskWqUvh7CLGW+UXxHQItqALa38kKnOiZPqJrtzJUgAC2+F0rL1S4Pg=="], + "@types/unist": ["@types/unist@3.0.3", "", {}, "sha512-ko/gIFJRv177XgZsZcBwnqJN5x/Gien8qNOn0D5bQU/zAzVf9Zt3BlcUiLqhV9y4ARk0GbT3tnUiPNgnTXzc/Q=="], "@types/web-bluetooth": ["@types/web-bluetooth@0.0.21", "", {}, "sha512-oIQLCGWtcFZy2JW77j9k8nHzAOpqMHLQejDA48XXMWH6tjCQHz5RCFz1bzsmROyL6PUm+LLnUiI4BCn221inxA=="], @@ -799,7 +809,7 @@ "body-parser": ["body-parser@2.3.0", "", { "dependencies": { "bytes": "^3.1.2", "content-type": "^2.0.0", "debug": "^4.4.3", "http-errors": "^2.0.1", "iconv-lite": "^0.7.2", "on-finished": "^2.4.1", "qs": "^6.15.2", "raw-body": "^3.0.2", "type-is": "^2.1.0" } }, "sha512-2cGmJupaNgg+QUwVLAucDuWuoMZ6EX9iHDRswZ5lsNYEmwPaRknMPCLZz07yTzVq/83p4o/wzbDZbBrTvGGTIw=="], - "boolbase": ["boolbase@1.0.0", "", {}, "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww=="], + "boolbase": ["boolbase@2.0.0", "", {}, "sha512-DkVaaQHymRhpYEYo9x1oo7Q7B0Y6KJUsjm3c9eTyFDby4MHLBTwZ6ZDWBel5zrYxj1WsZgC5oLpiz+93MluXeA=="], "brace-expansion": ["brace-expansion@5.0.9", "", { "dependencies": { "balanced-match": "^4.0.2" } }, "sha512-ScQ4IuvIEF1TMlP7Zt+vjJ//9zlPb2SDcxWxM3bk8s6t6GGdJ7KO1dCcTidOPJKePW30LE/2cT7wCyPho9/Wxg=="], @@ -901,11 +911,11 @@ "crossws": ["crossws@0.3.5", "", { "dependencies": { "uncrypto": "^0.1.3" } }, "sha512-ojKiDvcmByhwa8YYqbQI/hg7MEU0NC03+pSdEq4ZUnZR9xXpwk7E43SMNGkn+JxJGPFtNvQ48+vV2p+P1ml5PA=="], - "css-select": ["css-select@5.2.2", "", { "dependencies": { "boolbase": "^1.0.0", "css-what": "^6.1.0", "domhandler": "^5.0.2", "domutils": "^3.0.1", "nth-check": "^2.0.1" } }, "sha512-TizTzUddG/xYLA3NXodFM0fSbNizXjOKhqiQQwvhlspadZokn1KDy0NZFS0wuEubIYAV5/c1/lAr0TaaFXEXzw=="], + "css-select": ["css-select@7.0.0", "", { "dependencies": { "boolbase": "^2.0.0", "css-what": "^8.0.0", "domhandler": "^6.0.1", "domutils": "^4.0.2", "nth-check": "^3.0.1" } }, "sha512-snmjEVXy+1LnwXdxhYvTMj1d9tOh4HxkA1YmoayVBeeyR2C14Pum7fcxJIm4SswYspVy866eYNwlH6xC3/VH5g=="], "css-tree": ["css-tree@3.2.1", "", { "dependencies": { "mdn-data": "2.27.1", "source-map-js": "^1.2.1" } }, "sha512-X7sjQzceUhu1u7Y/ylrRZFU2FS6LRiFVp6rKLPg23y3x3c3DOKAwuXGDp+PAGjh6CSnCjYeAul8pcT8bAl+lSA=="], - "css-what": ["css-what@6.2.2", "", {}, "sha512-u/O3vwbptzhMs3L1fQE82ZSLHQQfto5gyZzwteVIEyeaY5Fc7R4dapF/BvRoSYFeqfBk4m0V1Vafq5Pjv25wvA=="], + "css-what": ["css-what@8.0.0", "", {}, "sha512-DH0Bqq3DNp5tdOReuNyAA+Ev4Y2GS5FMbZpeTLP6C4CDi0h5nL0BmUPChXw3o/qbHLDWHl49sbNqQVY7bMSDdw=="], "cssesc": ["cssesc@3.0.0", "", { "bin": { "cssesc": "bin/cssesc" } }, "sha512-/Tb/JcjK111nNScGob5MNtsntNM1aCNUDipB/TkwZFhyDrrE47SOx/18wF2bbjgc3ZzCSKW1T5nt5EbFoAz/Vg=="], @@ -917,6 +927,8 @@ "csso": ["csso@5.0.5", "", { "dependencies": { "css-tree": "~2.2.0" } }, "sha512-0LrrStPOdJj+SPCCrGhzryycLjwcgUSHBtxNA8aIDxf0GLsRh1cKYhB00Gd1lDOS4yGH69+SNn13+TWbVHETFQ=="], + "cssom": ["cssom@0.5.0", "", {}, "sha512-iKuQcq+NdHqlAcwUY0o/HL69XQrUaQdMjmStJ8JFmUaiiQErlhrmuigkg/CU4E2J0IyUKUrMAgl36TvN67MqTw=="], + "csstype": ["csstype@3.2.3", "", {}, "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ=="], "db0": ["db0@0.3.4", "", { "peerDependencies": { "@electric-sql/pglite": "*", "@libsql/client": "*", "better-sqlite3": "*", "drizzle-orm": "*", "mysql2": "*", "sqlite3": "*" }, "optionalPeers": ["@electric-sql/pglite", "@libsql/client", "better-sqlite3", "drizzle-orm", "mysql2", "sqlite3"] }, "sha512-RiXXi4WaNzPTHEOu8UPQKMooIbqOEyqA1t7Z6MsdxSCeb8iUC9ko3LcmsLmeUt2SM5bctfArZKkRQggKZz7JNw=="], @@ -951,13 +963,13 @@ "diff": ["diff@9.0.0", "", {}, "sha512-svtcdpS8CgJyqAjEQIXdb3OjhFVVYjzGAPO8WGCmRbrml64SPw/jJD4GoE98aR7r25A0XcgrK3F02yw9R/vhQw=="], - "dom-serializer": ["dom-serializer@2.0.0", "", { "dependencies": { "domelementtype": "^2.3.0", "domhandler": "^5.0.2", "entities": "^4.2.0" } }, "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg=="], + "dom-serializer": ["dom-serializer@3.1.1", "", { "dependencies": { "domelementtype": "^3.0.0", "domhandler": "^6.0.0", "entities": "^8.0.0" } }, "sha512-4MEa38/QexBob6gFNwu+EGdWvhJ1OKuNwdYY3Y3NyeWDQfnGeDYQUDfIRzWu5B5gsv03so2Uxd28YC6zrsx3Lw=="], "domelementtype": ["domelementtype@2.3.0", "", {}, "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw=="], - "domhandler": ["domhandler@5.0.3", "", { "dependencies": { "domelementtype": "^2.3.0" } }, "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w=="], + "domhandler": ["domhandler@6.0.1", "", { "dependencies": { "domelementtype": "^3.0.0" } }, "sha512-gYzvtM72ZtxQO0T048kd6HWSbbGCNOUwcnfQ01cqIJ4X2IYKFFHZ5mKvrQETcFXxsRObZulDaKmy//R7TPtsBg=="], - "domutils": ["domutils@3.2.2", "", { "dependencies": { "dom-serializer": "^2.0.0", "domelementtype": "^2.3.0", "domhandler": "^5.0.3" } }, "sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw=="], + "domutils": ["domutils@4.0.2", "", { "dependencies": { "dom-serializer": "^3.0.0", "domelementtype": "^3.0.0", "domhandler": "^6.0.0" } }, "sha512-qI4JLRKnSzqFqr7hAlS5xQDusBCjKSEG4t4+7aNrIQMHBcsC2TGEhuyABJdYkgSewL57PNLYEiibY2iPKhKpaA=="], "dot-prop": ["dot-prop@10.2.0", "", { "dependencies": { "type-fest": "^5.0.0" } }, "sha512-BTJ9aZYL3vCfZlZOBLy9v8TUqWGQ0pzFnygKwFZt5udj6viBoFIBviKPUoZLDCPn1FoXffv6McQFDenrm5Krfw=="], @@ -1143,8 +1155,12 @@ "hookable": ["hookable@6.1.1", "", {}, "sha512-U9LYDy1CwhMCnprUfeAZWZGByVbhd54hwepegYTK7Pi5NvqEj63ifz5z+xukznehT7i6NIZRu89Ay1AZmRsLEQ=="], + "html-escaper": ["html-escaper@3.0.3", "", {}, "sha512-RuMffC89BOWQoY0WKGpIhn5gX3iI54O6nRA0yC124NYVtzjmFWBIiFd8M0x+ZdX0P9R4lADg1mgP8C7PxGOWuQ=="], + "html-void-elements": ["html-void-elements@3.0.0", "", {}, "sha512-bEqo66MRXsUGxWHV5IP0PUiAWwoEjba4VCzg0LjFJBpchPaTfyfCKTG6bc5F8ucKec3q5y6qOdGyYTSBEvhCrg=="], + "htmlparser2": ["htmlparser2@10.1.0", "", { "dependencies": { "domelementtype": "^2.3.0", "domhandler": "^5.0.3", "domutils": "^3.2.2", "entities": "^7.0.1" } }, "sha512-VTZkM9GWRAtEpveh7MSF6SjjrpNVNNVJfFup7xTY3UpFtm67foy9HDVXneLtFVt4pMz5kZtgNcvCniNFb1hlEQ=="], + "http-errors": ["http-errors@2.0.1", "", { "dependencies": { "depd": "~2.0.0", "inherits": "~2.0.4", "setprototypeof": "~1.2.0", "statuses": "~2.0.2", "toidentifier": "~1.0.1" } }, "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ=="], "http-shutdown": ["http-shutdown@1.2.2", "", {}, "sha512-S9wWkJ/VSY9/k4qcjG318bqJNruzE4HySUhFYknwmu6LBP97KLLfwNf+n4V1BHurvFNkSKLFnK/RsuUnRTf9Vw=="], @@ -1295,6 +1311,8 @@ "lilconfig": ["lilconfig@3.1.3", "", {}, "sha512-/vlFKAoH5Cgt3Ie+JLhRbwOsCQePABiU3tJ1egGvyQ+33R/vcwM2Zl2QR/LzjsBeItPt3oSVXapn+m4nQDvpzw=="], + "linkedom": ["linkedom@0.18.13", "", { "dependencies": { "css-select": "^7.0.0", "cssom": "^0.5.0", "html-escaper": "^3.0.3", "htmlparser2": "^10.1.0", "uhyphen": "^0.2.0" }, "peerDependencies": { "canvas": ">= 2" }, "optionalPeers": ["canvas"] }, "sha512-ES/o9qotMpzpN2MHs+Iq/JcVoOj8Fa5wiQYrTdFpvAnwXL0g66XHHUc9WUMk6nAlBtGsFQ24ne+SYnvnaQ2FSw=="], + "listhen": ["listhen@1.10.1", "", { "dependencies": { "@parcel/watcher-wasm": "^2.5.6", "citty": "^0.2.2", "consola": "^3.4.2", "crossws": "^0.4.10", "defu": "^6.1.7", "get-port-please": "^3.2.0", "h3": "^1.15.11", "http-shutdown": "^1.2.2", "jiti": "^2.7.0", "node-forge": "^1.4.0", "pathe": "^2.0.3", "std-env": "^4.2.0", "tinyclip": "^0.1.15", "ufo": "^1.6.4", "untun": "^0.2.2", "uqr": "^0.1.3" }, "peerDependencies": { "@parcel/watcher": "^2.5.6" }, "optionalPeers": ["@parcel/watcher"], "bin": { "listen": "./bin/listhen.mjs", "listhen": "./bin/listhen.mjs" } }, "sha512-6nt/86SkqUQSLW1ofz8MxC6RhRMqOl3ONISe6qqvJ3xj09aJWQx6DhgSZpugs3PX4PXdOas/WD6A9jx6J2N19A=="], "loader-utils": ["loader-utils@3.3.1", "", {}, "sha512-FMJTLMXfCLMLfJxcX9PFqX5qD88Z5MRGaZCVzfuqeZSPsyiBzs+pahDQjbIWz2QIzPZz0NX9Zy4FX3lmK6YHIg=="], @@ -1409,7 +1427,7 @@ "npm-run-path": ["npm-run-path@5.3.0", "", { "dependencies": { "path-key": "^4.0.0" } }, "sha512-ppwTtiJZq0O/ai0z7yfudtBpWIoxM8yE6nHi1X47eFR2EWORqfbu6CnPlNsjeN683eT0qG6H/Pyf9fCcvjnnnQ=="], - "nth-check": ["nth-check@2.1.1", "", { "dependencies": { "boolbase": "^1.0.0" } }, "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w=="], + "nth-check": ["nth-check@3.0.1", "", { "dependencies": { "boolbase": "^2.0.0" } }, "sha512-GX0gsdbGVCgnRgbeGaubfjpBXyYRWOOCVeYh08bSQvDZqxz5ndXs1OTfAt/h36G1xvI94YIspsI0sVFqAV9+RQ=="], "nuxt": ["nuxt@4.5.2", "", { "dependencies": { "@dxup/nuxt": "^0.5.6", "@nuxt/cli": "^3.37.0", "@nuxt/devtools": "^3.4.1", "@nuxt/kit": "4.5.2", "@nuxt/nitro-server": "4.5.2", "@nuxt/schema": "4.5.2", "@nuxt/telemetry": "^2.8.0", "@nuxt/vite-builder": "4.5.2", "@unhead/vue": "^3.3.1", "@vue/shared": "^3.5.40", "chokidar": "^5.0.0", "compatx": "^0.2.0", "consola": "^3.4.2", "cookie-es": "^3.1.1", "defu": "^6.1.7", "devalue": "^5.9.0", "errx": "^0.1.2", "escape-string-regexp": "^5.0.0", "exsolve": "^1.1.1", "fnv1a-64": "^0.1.2", "hookable": "^6.1.1", "ignore": "^7.0.6", "impound": "^1.1.6", "jiti": "^2.7.0", "klona": "^2.0.6", "knitwork": "^1.3.0", "magic-string": "^1.1.0", "mlly": "^1.8.2", "nanotar": "^0.3.0", "nostics": "^1.2.0", "nypm": "^0.6.9", "object-identity": "^0.2.3", "ofetch": "^1.5.1", "ohash": "^2.0.11", "on-change": "^6.0.2", "oxc-walker": "^1.1.1", "pathe": "^2.0.3", "perfect-debounce": "^2.1.0", "picomatch": "^4.0.5", "pkg-types": "^2.3.1", "rolldown-string": "^0.3.1", "rou3": "^0.9.1", "scule": "^1.3.0", "std-env": "^4.2.0", "tinyglobby": "^0.2.17", "ufo": "^1.6.4", "ultrahtml": "^1.7.0", "uncrypto": "^0.1.3", "unctx": "^3.0.0", "undici": "^8.10.0", "unhead": "^3.3.1", "unimport": "^6.4.0", "unplugin": "^3.3.0", "unrouting": "^0.2.2", "untyped": "^2.0.0", "verkit": "^0.3.1", "vue": "^3.5.40", "vue-component-type-helpers": "^3.3.9", "vue-router": "^5.2.0" }, "peerDependencies": { "@parcel/watcher": "^2.1.0", "@types/node": ">=18.12.0", "rolldown": "~1.2.1" }, "optionalPeers": ["@parcel/watcher", "@types/node"], "bin": { "nuxi": "bin/nuxt.mjs", "nuxt": "bin/nuxt.mjs" } }, "sha512-tR3fcqeHlHmmkLMpIg3V7Y+1ltr302lW8djMw/iy+myfo7QSSz+BVJDuQhg5j73b9oteSyBfOKTDYTgvMtj6TA=="], @@ -1785,6 +1803,8 @@ "tslib": ["tslib@2.8.1", "", {}, "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w=="], + "turndown": ["turndown@7.2.4", "", { "dependencies": { "@mixmark-io/domino": "^2.2.0" } }, "sha512-I8yFsfRzmzK0WV1pNNOA4A7y4RDfFxPRxb3t+e3ui14qSGOxGtiSP6GjeX+Y6CHb7HYaFj7ECUD7VE5kQMZWGQ=="], + "type-fest": ["type-fest@5.8.0", "", { "dependencies": { "tagged-tag": "^1.0.0" } }, "sha512-YGYEVz3Fm5iy/AybuA0oyNFq7H4CgQNfRp/qfe8nurE1kuCeNm3/vfm9X4Mtl+qLyaKJUh5xrFZwogr41SMjYA=="], "type-is": ["type-is@2.1.0", "", { "dependencies": { "content-type": "^2.0.0", "media-typer": "^1.1.0", "mime-types": "^3.0.0" } }, "sha512-faYHw0anBbc/kWF3zFTEnxSFOAGUX9GFbOBthvDdLsIlEoWOFOtS0zgCiQYwIskL9iGXZL3kAXD8OoZ4GmMATA=="], @@ -1799,6 +1819,8 @@ "ufo": ["ufo@1.6.4", "", {}, "sha512-JFNbkD1Svwe0KvGi8GOeLcP4kAWQ609twvCdcHxq1oSL8svv39ZuSvajcD8B+5D0eL4+s1Is2D/O6KN3qcTeRA=="], + "uhyphen": ["uhyphen@0.2.0", "", {}, "sha512-qz3o9CHXmJJPGBdqzab7qAYuW8kQGKNEuoHFYrBwV6hWIMcpAmxDLXojcHfFr9US1Pe6zUswEIJIbLI610fuqA=="], + "uint8array-extras": ["uint8array-extras@1.5.0", "", {}, "sha512-rvKSBiC5zqCCiDZ9kAOszZcDvdAHwwIKJG33Ykj43OKcWsnmcBRL09YTU4nOeHZ8Y2a7l1MgTd08SBe9A8Qj6A=="], "ultrahtml": ["ultrahtml@1.7.0", "", {}, "sha512-2xRd0VHoAQE4M+vF/DvFFB7pUV0ZxTW1TLi7lHQWnF/Sb5TPeEUV/l+hxcNnGO00ZXGnR0voCMmYRKQf+rvJ2g=="], @@ -2065,7 +2087,13 @@ "csso/css-tree": ["css-tree@2.2.1", "", { "dependencies": { "mdn-data": "2.0.28", "source-map-js": "^1.0.1" } }, "sha512-OA0mILzGc1kCOCSJerOeqDxDQ4HOh+G8NbOJFOTgOCzpw7fCBubk0fEyxp8AgOL/jvLgYA/uV0cMbe43ElF1JA=="], - "dom-serializer/entities": ["entities@4.5.0", "", {}, "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw=="], + "dom-serializer/domelementtype": ["domelementtype@3.0.0", "", {}, "sha512-umCQid3jKbDmVjx8jGaW7uUykm4DEUeyV21hPxNMo2nV955DhUThwqyOIDtreepP31hl84X7G5U9ZfsWvIB3Pg=="], + + "dom-serializer/entities": ["entities@8.0.0", "", {}, "sha512-zwfzJecQ/Uej6tusMqwAqU/6KL2XaB2VZ2Jg54Je6ahNBGNH6Ek6g3jjNCF0fG9EWQKGZNddNjU5F1ZQn/sBnA=="], + + "domhandler/domelementtype": ["domelementtype@3.0.0", "", {}, "sha512-umCQid3jKbDmVjx8jGaW7uUykm4DEUeyV21hPxNMo2nV955DhUThwqyOIDtreepP31hl84X7G5U9ZfsWvIB3Pg=="], + + "domutils/domelementtype": ["domelementtype@3.0.0", "", {}, "sha512-umCQid3jKbDmVjx8jGaW7uUykm4DEUeyV21hPxNMo2nV955DhUThwqyOIDtreepP31hl84X7G5U9ZfsWvIB3Pg=="], "elysia/memoirist": ["memoirist@0.4.0", "", {}, "sha512-zxTgA0mSYELa66DimuNQDvyLq36AwDlTuVRbnQtB+VuTcKWm5Qc4z3WkSpgsFWHNhexqkIooqpv4hdcqrX5Nmg=="], @@ -2083,6 +2111,10 @@ "h3/cookie-es": ["cookie-es@1.2.3", "", {}, "sha512-lXVyvUvrNXblMqzIRrxHb57UUVmqsSWlxqt3XIjCkUP0wDAf6uicO6KMbEgYrMNtEvWgWHwe42CKxPu9MYAnWw=="], + "htmlparser2/domhandler": ["domhandler@5.0.3", "", { "dependencies": { "domelementtype": "^2.3.0" } }, "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w=="], + + "htmlparser2/domutils": ["domutils@3.2.2", "", { "dependencies": { "dom-serializer": "^2.0.0", "domelementtype": "^2.3.0", "domhandler": "^5.0.3" } }, "sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw=="], + "is-inside-container/is-docker": ["is-docker@3.0.0", "", { "bin": { "is-docker": "cli.js" } }, "sha512-eljcgEDlEns/7AXFosB5K/2nCM4P7FQPkGc/DWLy5rmFEWvZayGrik1d9/QIY5nJ4f9YsVvBkA6kJpHn9rISdQ=="], "lazystream/readable-stream": ["readable-stream@2.3.8", "", { "dependencies": { "core-util-is": "~1.0.0", "inherits": "~2.0.3", "isarray": "~1.0.0", "process-nextick-args": "~2.0.0", "safe-buffer": "~5.1.1", "string_decoder": "~1.1.1", "util-deprecate": "~1.0.1" } }, "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA=="], @@ -2157,6 +2189,10 @@ "svgo/commander": ["commander@11.1.0", "", {}, "sha512-yPVavfyCcRhmorC7rWlkHn15b4wDVgVmBA7kV4QVBsF7kv/9TKJAbAXVTxvTnwP8HHKjRCJDClKbciiYS7p0DQ=="], + "svgo/css-select": ["css-select@5.2.2", "", { "dependencies": { "boolbase": "^1.0.0", "css-what": "^6.1.0", "domhandler": "^5.0.2", "domutils": "^3.0.1", "nth-check": "^2.0.1" } }, "sha512-TizTzUddG/xYLA3NXodFM0fSbNizXjOKhqiQQwvhlspadZokn1KDy0NZFS0wuEubIYAV5/c1/lAr0TaaFXEXzw=="], + + "svgo/css-what": ["css-what@6.2.2", "", {}, "sha512-u/O3vwbptzhMs3L1fQE82ZSLHQQfto5gyZzwteVIEyeaY5Fc7R4dapF/BvRoSYFeqfBk4m0V1Vafq5Pjv25wvA=="], + "terser/commander": ["commander@2.20.3", "", {}, "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ=="], "type-is/content-type": ["content-type@2.1.0", "", {}, "sha512-mj7UPXE0jaqaOsukNZRUEfEi2AcL7C/vwmwcHV0O97eO1E1pxBZuyjlZrx5seTaNBg1U6+o35wpa35Qfcc+7ag=="], @@ -2289,6 +2325,8 @@ "fontless/esbuild/@esbuild/win32-x64": ["@esbuild/win32-x64@0.27.7", "", { "os": "win32", "cpu": "x64" }, "sha512-56hiAJPhwQ1R4i+21FVF7V8kSD5zZTdHcVuRFMW0hn753vVfQN8xlx4uOPT4xoGH0Z/oVATuR82AiqSTDIpaHg=="], + "htmlparser2/domutils/dom-serializer": ["dom-serializer@2.0.0", "", { "dependencies": { "domelementtype": "^2.3.0", "domhandler": "^5.0.2", "entities": "^4.2.0" } }, "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg=="], + "lazystream/readable-stream/safe-buffer": ["safe-buffer@5.1.2", "", {}, "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g=="], "lazystream/readable-stream/string_decoder": ["string_decoder@1.1.1", "", { "dependencies": { "safe-buffer": "~5.1.0" } }, "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg=="], @@ -2371,6 +2409,14 @@ "readdir-glob/minimatch/brace-expansion": ["brace-expansion@2.1.4", "", { "dependencies": { "balanced-match": "^1.0.0" } }, "sha512-hGfVzPxthbf3+2yjg/RBs60cB0FhqBS/zvdV/4wn4/BmN0bNMMHPc4V/BbFieqf1TKAGGAHnY4eSjajCl0f2Xg=="], + "svgo/css-select/boolbase": ["boolbase@1.0.0", "", {}, "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww=="], + + "svgo/css-select/domhandler": ["domhandler@5.0.3", "", { "dependencies": { "domelementtype": "^2.3.0" } }, "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w=="], + + "svgo/css-select/domutils": ["domutils@3.2.2", "", { "dependencies": { "dom-serializer": "^2.0.0", "domelementtype": "^2.3.0", "domhandler": "^5.0.3" } }, "sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw=="], + + "svgo/css-select/nth-check": ["nth-check@2.1.1", "", { "dependencies": { "boolbase": "^1.0.0" } }, "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w=="], + "vite-node/vite/fsevents": ["fsevents@2.3.3", "", { "os": "darwin" }, "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw=="], "vite-plugin-checker/npm-run-path/path-key": ["path-key@4.0.0", "", {}, "sha512-haREypq7xkM7ErfgIyA0z+Bj4AGKlMSdlQE2jvJo6huWD1EdkKYV+G/T4nq0YEF2vgTT8kqMFKo1uHn950r4SQ=="], @@ -2401,12 +2447,16 @@ "archiver-utils/glob/path-scurry/lru-cache": ["lru-cache@10.4.3", "", {}, "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ=="], + "htmlparser2/domutils/dom-serializer/entities": ["entities@4.5.0", "", {}, "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw=="], + "nuxtseo-shared/@nuxt/devtools-kit/@nuxt/kit/verkit": ["verkit@0.2.0", "", {}, "sha512-6M8R2xSKplkQ+0mAm07IMh548GLLPUnRQZJCCe/W4rfk/SXW2bs8ZJSWa5kjdIdsx3hLG5oLWROJ4+N5hpX08g=="], "nuxtseo-shared/@nuxt/devtools-kit/vite/fsevents": ["fsevents@2.3.3", "", { "os": "darwin" }, "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw=="], "readdir-glob/minimatch/brace-expansion/balanced-match": ["balanced-match@1.0.2", "", {}, "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw=="], + "svgo/css-select/domutils/dom-serializer": ["dom-serializer@2.0.0", "", { "dependencies": { "domelementtype": "^2.3.0", "domhandler": "^5.0.2", "entities": "^4.2.0" } }, "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg=="], + "vue-router/@vue/devtools-api/@vue/devtools-kit/@vue/devtools-shared": ["@vue/devtools-shared@8.2.1", "", {}, "sha512-Fkac7lUdGReh6pVOi3AYPRGe82LQqRmAfThW7RRligOAP0ZA/Z1z9XLHDM9dv34pV2HRc79DK8uKPeG2fLnA/g=="], "vue-router/@vue/devtools-api/@vue/devtools-kit/birpc": ["birpc@2.9.0", "", {}, "sha512-KrayHS5pBi69Xi9JmvoqrIgYGDkD6mcSe/i6YKi3w5kekCLzrX4+nawcXqrj2tIp50Kw/mT/s3p+GVK0A0sKxw=="], @@ -2420,5 +2470,7 @@ "@nuxtjs/sitemap/nuxtseo-shared/@nuxt/devtools-kit/vite/fsevents": ["fsevents@2.3.3", "", { "os": "darwin" }, "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw=="], "archiver-utils/glob/minimatch/brace-expansion/balanced-match": ["balanced-match@1.0.2", "", {}, "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw=="], + + "svgo/css-select/domutils/dom-serializer/entities": ["entities@4.5.0", "", {}, "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw=="], } } From 4d4d2d5b8635816897e5eaadb44760dd0ed45aaf Mon Sep 17 00:00:00 2001 From: germondai Date: Fri, 4 Sep 2026 02:41:29 +0200 Subject: [PATCH 2/4] feat(mcp): add agent-focused web tools --- apps/api/src/routes/mcp.ts | 309 ++++++++++++++++++++++++++++++++----- 1 file changed, 274 insertions(+), 35 deletions(-) diff --git a/apps/api/src/routes/mcp.ts b/apps/api/src/routes/mcp.ts index 8801e5f..1d232ba 100644 --- a/apps/api/src/routes/mcp.ts +++ b/apps/api/src/routes/mcp.ts @@ -1,9 +1,12 @@ import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js" import { WebStandardStreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/webStandardStreamableHttp.js" +import { Readability } from "@mozilla/readability" import { PoolExhaustedError } from "@trawl/browser" import { RequestValidationError, ScrapeError, scrape } from "@trawl/tiers" -import type { ScrapeResult } from "@trawl/types" +import type { ScrapeRequest, ScrapeResult } from "@trawl/types" import { Elysia } from "elysia" +import { parseHTML } from "linkedom" +import TurndownService from "turndown" import * as z from "zod/v4" import pkg from "../../package.json" import { MCP_ALLOWED_ORIGINS } from "../config" @@ -11,13 +14,13 @@ import { getDeps, getPool } from "../deps" import { assertPublicHttpUrl, createPublicUrlValidator } from "../outbound-policy" export const MCP_HTML_MAX_CHARS = 50_000 +export const MCP_READ_MAX_CHARS = 100_000 -type RunScrape = (input: { - url: string - maxTimeout?: number - maxTier?: 1 | 2 | 3 | 4 - skipHttp?: boolean -}) => Promise +type McpScrapeInput = Pick< + ScrapeRequest, + "url" | "maxTimeout" | "maxTier" | "skipHttp" | "screenshot" | "consoleLogs" | "networkLogs" | "redirectChain" +> +type RunScrape = (input: McpScrapeInput) => Promise interface McpRouteOptions { allowedOrigins?: string[] @@ -25,6 +28,22 @@ interface McpRouteOptions { runScrape?: RunScrape } +const tierSchema = z.union([z.literal(1), z.literal(2), z.literal(3), z.literal(4)]) +const browserTierSchema = z.union([z.literal(2), z.literal(3), z.literal(4)]) +const baseInputShape = { + url: z.string().min(1).describe("Public HTTP(S) URL"), + maxTimeout: z.number().int().positive().optional().describe("Maximum operation time in milliseconds"), + maxTier: tierSchema.optional().describe("Highest anti-bot tier TRAWL may use"), +} +const metadataSchema = { + url: z.string(), + statusCode: z.number().int(), + tier: tierSchema, + contentType: z.string(), + totalMs: z.number(), +} +const toolAnnotations = { readOnlyHint: true, openWorldHint: true } as const + function errorResult(error: unknown) { let message = error instanceof Error ? error.message : String(error) if (error instanceof PoolExhaustedError) message = "Browser pool saturated, retry shortly" @@ -32,53 +51,273 @@ function errorResult(error: unknown) { return { content: [{ type: "text" as const, text: message }], isError: true } } +function contentType(result: ScrapeResult): string { + return result.contentType ?? result.responseHeaders?.["content-type"] ?? "text/html" +} + +function metadata(result: ScrapeResult) { + return { + url: result.url, + statusCode: result.statusCode, + tier: result.tier, + contentType: contentType(result), + totalMs: result.totalMs, + } +} + +function redactUrl(rawUrl: string): string { + try { + const url = new URL(rawUrl) + url.username = "" + url.password = "" + url.search = "" + url.hash = "" + return url.toString() + } catch { + return "[invalid URL]" + } +} + +function extractReadable(html: string, url: string, format: "markdown" | "text") { + const { document } = parseHTML(html) + const parsed = new Readability(document as unknown as Document).parse() + const title = parsed?.title?.trim() || document.title?.trim() || url + const articleHtml = parsed?.content || document.body?.innerHTML || html + const text = + format === "markdown" + ? new TurndownService({ headingStyle: "atx", codeBlockStyle: "fenced" }).turndown(articleHtml) + : parsed?.textContent?.trim() || document.body?.textContent?.trim() || "" + return { + text, + title, + ...(parsed?.byline ? { byline: parsed.byline } : {}), + ...(parsed?.excerpt ? { excerpt: parsed.excerpt } : {}), + ...(parsed?.siteName ? { siteName: parsed.siteName } : {}), + ...(parsed?.lang ? { language: parsed.lang } : {}), + } +} + function createServer(poolReady: () => boolean, runScrape: RunScrape): McpServer { - const server = new McpServer({ name: "trawl", version: pkg.version }) + const server = new McpServer( + { name: "trawl", version: pkg.version }, + { + instructions: + "Use read for concise article content, scrape for source HTML, screenshot for visual inspection, and inspect for browser diagnostics. TRAWL fetches known URLs; it does not search the web.", + }, + ) + + const runSafe = async (input: McpScrapeInput) => { + await assertPublicHttpUrl(input.url) + if (!poolReady()) throw new Error("Browser pool initializing, retry in a few seconds") + const result = await runScrape(input) + await assertPublicHttpUrl(result.url) + return result + } + + const scrapeHandler = async (input: { + url: string + maxTimeout?: number + maxTier?: 1 | 2 | 3 | 4 + skipHttp?: boolean + }) => { + try { + const result = await runSafe(input) + const truncated = result.html.length > MCP_HTML_MAX_CHARS + const html = result.html.slice(0, MCP_HTML_MAX_CHARS) + const structuredContent = { + ...metadata(result), + truncated, + sessionCached: result.sessionCached, + timings: result.timings, + ...(result.captchasSolved ? { captchasSolved: result.captchasSolved } : {}), + ...(result.proxyUsed === undefined ? {} : { proxyUsed: result.proxyUsed }), + } + return { content: [{ type: "text" as const, text: html }], structuredContent } + } catch (error) { + return errorResult(error) + } + } + + const scrapeConfig = { + title: "Scrape page", + description: + "Fetch source HTML from a known public URL through TRAWL's anti-bot tiers. Prefer read when the user wants page content rather than markup.", + inputSchema: z.strictObject({ + ...baseInputShape, + skipHttp: z.boolean().optional().describe("Skip the plain HTTP tier and start with a browser"), + }), + outputSchema: { + ...metadataSchema, + truncated: z.boolean(), + sessionCached: z.boolean(), + timings: z.array( + z.object({ + tier: tierSchema, + status: z.enum(["success", "blocked", "needs-js", "timeout", "error", "skipped"]), + durationMs: z.number(), + reason: z.string().optional(), + }), + ), + captchasSolved: z.array(z.string()).optional(), + proxyUsed: z.boolean().optional(), + }, + annotations: toolAnnotations, + } + server.registerTool("scrape", scrapeConfig, scrapeHandler) server.registerTool( "scrape_url", { - title: "Scrape URL", - description: "Fetch a known public HTTP(S) URL, escalating through TRAWL's anti-bot tiers when necessary.", + ...scrapeConfig, + title: "Scrape URL (compatibility alias)", + description: "Compatibility alias for scrape. Fetch source HTML from a known public URL.", + }, + scrapeHandler, + ) + + server.registerTool( + "read", + { + title: "Read page", + description: + "Extract the main human-readable content from a known public URL. Use this for articles, documentation and research.", inputSchema: z.strictObject({ - url: z.string().min(1).describe("Public HTTP(S) URL to scrape"), - maxTimeout: z.number().int().positive().optional().describe("Maximum scrape time in milliseconds"), - maxTier: z.union([z.literal(1), z.literal(2), z.literal(3), z.literal(4)]).optional(), - skipHttp: z.boolean().optional().describe("Skip the plain HTTP tier"), + ...baseInputShape, + format: z.enum(["markdown", "text"]).optional().describe("Output format; defaults to markdown"), + maxCharacters: z + .number() + .int() + .min(1) + .max(MCP_READ_MAX_CHARS) + .optional() + .describe(`Maximum returned characters; defaults to ${MCP_HTML_MAX_CHARS}`), }), outputSchema: { - url: z.string(), - statusCode: z.number().int(), - tier: z.number().int(), - contentType: z.string(), - totalMs: z.number(), + ...metadataSchema, + title: z.string(), + format: z.enum(["markdown", "text"]), + characters: z.number().int(), truncated: z.boolean(), + byline: z.string().optional(), + excerpt: z.string().optional(), + siteName: z.string().optional(), + language: z.string().optional(), }, - annotations: { readOnlyHint: true, openWorldHint: true }, + annotations: toolAnnotations, + }, + async ({ format = "markdown", maxCharacters = MCP_HTML_MAX_CHARS, ...input }) => { + try { + const result = await runSafe(input) + const readable = extractReadable(result.html, result.url, format) + const truncated = readable.text.length > maxCharacters + const text = readable.text.slice(0, maxCharacters) + return { + content: [{ type: "text", text }], + structuredContent: { + ...metadata(result), + title: readable.title, + format, + characters: text.length, + truncated, + ...(readable.byline ? { byline: readable.byline } : {}), + ...(readable.excerpt ? { excerpt: readable.excerpt } : {}), + ...(readable.siteName ? { siteName: readable.siteName } : {}), + ...(readable.language ? { language: readable.language } : {}), + }, + } + } catch (error) { + return errorResult(error) + } + }, + ) + + server.registerTool( + "screenshot", + { + title: "Screenshot page", + description: + "Render a known public URL in TRAWL's browser and return a viewport JPEG for visual or multimodal inspection.", + inputSchema: z.strictObject({ + ...baseInputShape, + maxTier: browserTierSchema.optional().describe("Highest browser tier TRAWL may use"), + }), + outputSchema: { ...metadataSchema, mimeType: z.literal("image/jpeg") }, + annotations: toolAnnotations, }, async (input) => { try { - await assertPublicHttpUrl(input.url) - if (!poolReady()) throw new Error("Browser pool initializing, retry in a few seconds") - const result = await runScrape(input) - // A redirect can reveal a policy violation even if the initial URL was public. - // Reject it from the model output as a second line of defense. - await assertPublicHttpUrl(result.url) - const truncated = result.html.length > MCP_HTML_MAX_CHARS - const html = result.html.slice(0, MCP_HTML_MAX_CHARS) + const result = await runSafe({ ...input, skipHttp: true, screenshot: true }) + if (!result.screenshot) throw new ScrapeError("Screenshot capture failed", result.timings) + return { + content: [{ type: "image", data: result.screenshot, mimeType: "image/jpeg" }], + structuredContent: { ...metadata(result), mimeType: "image/jpeg" as const }, + } + } catch (error) { + return errorResult(error) + } + }, + ) + + server.registerTool( + "inspect", + { + title: "Inspect page", + description: + "Load a known public URL in a browser and return bounded console, network and redirect diagnostics. Use for web debugging, not ordinary reading.", + inputSchema: z.strictObject({ + ...baseInputShape, + maxTier: browserTierSchema.optional().describe("Highest browser tier TRAWL may use"), + }), + outputSchema: { + ...metadataSchema, + consoleLogs: z.array( + z.object({ + level: z.enum(["SEVERE", "WARNING", "INFO", "DEBUG"]), + message: z.string(), + timestamp: z.number(), + source: z.string(), + }), + ), + networkLogs: z.array( + z.object({ + name: z.string(), + entryType: z.enum(["navigation", "resource"]), + startTime: z.number(), + duration: z.number(), + initiatorType: z.string(), + transferSize: z.number().nullable(), + encodedBodySize: z.number().nullable(), + decodedBodySize: z.number().nullable(), + }), + ), + redirectChain: z.array(z.string()), + }, + annotations: toolAnnotations, + }, + async (input) => { + try { + const result = await runSafe({ + ...input, + skipHttp: true, + consoleLogs: true, + networkLogs: true, + redirectChain: true, + }) const structuredContent = { - url: result.url, - statusCode: result.statusCode, - tier: result.tier, - contentType: result.contentType ?? result.responseHeaders?.["content-type"] ?? "text/html", - totalMs: result.totalMs, - truncated, + ...metadata(result), + consoleLogs: result.consoleLogs ?? [], + networkLogs: (result.networkLogs ?? []).map((entry) => ({ ...entry, name: redactUrl(entry.name) })), + redirectChain: (result.redirectChain ?? []).map(redactUrl), + } + return { + content: [{ type: "text", text: JSON.stringify(structuredContent, null, 2) }], + structuredContent, } - return { content: [{ type: "text", text: html }], structuredContent } } catch (error) { return errorResult(error) } }, ) + return server } From 19b4abcdeb3deca6932a50f8d535236bec55bc48 Mon Sep 17 00:00:00 2001 From: germondai Date: Fri, 4 Sep 2026 02:41:29 +0200 Subject: [PATCH 3/4] test(mcp): cover agent tool workflows --- apps/api/src/app.test.ts | 2 +- apps/api/src/routes/mcp.test.ts | 90 ++++++++++++++++++++++++++++++++- 2 files changed, 89 insertions(+), 3 deletions(-) diff --git a/apps/api/src/app.test.ts b/apps/api/src/app.test.ts index 6050f24..d5572de 100644 --- a/apps/api/src/app.test.ts +++ b/apps/api/src/app.test.ts @@ -24,7 +24,7 @@ describe("MCP opt-in registration", () => { try { await client.connect(new StreamableHTTPClientTransport(new URL(`http://127.0.0.1:${server.port}/mcp`))) const tools = await client.listTools() - expect(tools.tools.map((tool) => tool.name)).toEqual(["scrape_url"]) + expect(tools.tools.map((tool) => tool.name)).toEqual(["scrape", "scrape_url", "read", "screenshot", "inspect"]) } finally { await client.close() server.stop(true) diff --git a/apps/api/src/routes/mcp.test.ts b/apps/api/src/routes/mcp.test.ts index 6736b74..36b3558 100644 --- a/apps/api/src/routes/mcp.test.ts +++ b/apps/api/src/routes/mcp.test.ts @@ -28,7 +28,7 @@ function rpc(method: string, params?: unknown, id = 1): Request { } describe("MCP route", () => { - test("initializes and lists only scrape_url", async () => { + test("initializes and lists the focused tool set with the compatibility alias", async () => { const app = mcpRoute({ poolReady: () => true, runScrape: async () => baseResult }) const initialized = await app.handle( rpc("initialize", { @@ -43,7 +43,13 @@ describe("MCP route", () => { const listed = await app.handle(rpc("tools/list")) expect(listed.status).toBe(200) const body = await listed.json() - expect(body.result.tools.map((tool: { name: string }) => tool.name)).toEqual(["scrape_url"]) + expect(body.result.tools.map((tool: { name: string }) => tool.name)).toEqual([ + "scrape", + "scrape_url", + "read", + "screenshot", + "inspect", + ]) }) test("exposes the Streamable HTTP GET channel", async () => { @@ -75,12 +81,92 @@ describe("MCP route", () => { contentType: "text/html; charset=utf-8", totalMs: 12, truncated: true, + sessionCached: true, + timings: baseResult.timings, }) expect(body.result.content[0].text.length).toBe(MCP_HTML_MAX_CHARS) expect(JSON.stringify(body)).not.toContain('cookie"') expect(JSON.stringify(body)).not.toContain("secret-agent") }) + test("extracts readable markdown with metadata and a caller-controlled limit", async () => { + const app = mcpRoute({ + poolReady: () => true, + runScrape: async () => ({ + ...baseResult, + html: `Ignored shell title

Useful title

${"Readable content ".repeat(40)}

`, + }), + }) + const response = await app.handle( + rpc("tools/call", { + name: "read", + arguments: { url: "https://1.1.1.1/article", format: "markdown", maxCharacters: 80 }, + }), + ) + const result = (await response.json()).result + expect(result.isError).toBeUndefined() + expect(result.content[0].text).toContain("Useful title") + expect(result.content[0].text.length).toBe(80) + expect(result.structuredContent).toMatchObject({ format: "markdown", characters: 80, truncated: true }) + }) + + test("returns screenshots as MCP image content and forces a browser tier", async () => { + let received: unknown + const app = mcpRoute({ + poolReady: () => true, + runScrape: async (input) => { + received = input + return { ...baseResult, screenshot: "aGVsbG8=" } + }, + }) + const response = await app.handle( + rpc("tools/call", { name: "screenshot", arguments: { url: "https://1.1.1.1", maxTier: 3 } }), + ) + const result = (await response.json()).result + expect(received).toEqual({ url: "https://1.1.1.1", maxTier: 3, skipHttp: true, screenshot: true }) + expect(result.content[0]).toEqual({ type: "image", data: "aGVsbG8=", mimeType: "image/jpeg" }) + expect(result.structuredContent.mimeType).toBe("image/jpeg") + }) + + test("returns browser diagnostics while redacting URL credentials and query strings", async () => { + let received: unknown + const app = mcpRoute({ + poolReady: () => true, + runScrape: async (input) => { + received = input + return { + ...baseResult, + consoleLogs: [{ level: "SEVERE", message: "boom", timestamp: 1, source: "error" }], + networkLogs: [ + { + name: "https://user:secret@example.com/api?token=secret#part", + entryType: "resource", + startTime: 1, + duration: 2, + initiatorType: "fetch", + transferSize: 3, + encodedBodySize: 2, + decodedBodySize: null, + }, + ], + redirectChain: ["https://example.com/start?token=secret", "https://example.com/final#private"], + } + }, + }) + const response = await app.handle(rpc("tools/call", { name: "inspect", arguments: { url: "https://1.1.1.1" } })) + const result = (await response.json()).result + expect(received).toEqual({ + url: "https://1.1.1.1", + skipHttp: true, + consoleLogs: true, + networkLogs: true, + redirectChain: true, + }) + expect(result.structuredContent.networkLogs[0].name).toBe("https://example.com/api") + expect(result.structuredContent.redirectChain).toEqual(["https://example.com/start", "https://example.com/final"]) + expect(JSON.stringify(result)).not.toContain("secret") + }) + test("rejects extra arguments, private targets, and disallowed origins", async () => { const app = mcpRoute({ allowedOrigins: ["https://chat.example"], From e556234a70a5cab9f4290a0a508d90cce9d9c581 Mon Sep 17 00:00:00 2001 From: germondai Date: Fri, 4 Sep 2026 02:41:29 +0200 Subject: [PATCH 4/4] docs(mcp): document agent-focused tools --- CHANGELOG.md | 1 + README.md | 9 +++-- apps/docs/getting-started/configuration.md | 7 ++-- apps/docs/integrations/mcp.md | 47 +++++++++++++++++----- 4 files changed, 46 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1eab5bd..0096492 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Renamed internal byte/character limits to `STREAM_THRESHOLD_BYTES` and `MCP_HTML_MAX_CHARS`, and namespaced the Camoufox font-retention build argument as `CAMOUFOX_KEEP_SPOOFED_OS_FONTS`. ### Added +- Expanded the optional MCP server with purpose-specific `read`, `scrape`, `screenshot`, and `inspect` tools. AI clients can now receive readability-extracted Markdown/text, native image content, richer scrape metadata, or bounded browser diagnostics without exposing cookies, sessions, request headers, proxy credentials, or captured API response bodies. The original `scrape_url` remains as a compatibility alias. - Optional `MITM_ALWAYS_SCRAPE=true` mode skips the forward proxy's direct Tier 0 probe and routes ordinary HTTP requests immediately into the existing scraper ladder for sites where the probe itself triggers a temporary ban (#93). WebSocket relays remain direct; the mode is off by default and documented as unsuitable for general media/download traffic because it bypasses Tier 0 streaming. - **DataDome support.** Detect Device Check, slider CAPTCHA and `t=bv` hard blocks from challenge markers and `x-dd-b`. Device Check uses a dedicated waiter and an optional headful Xvfb pool; the slider is reported as `datadome-captcha-required`. Enable startup-warmed capacity with `BROWSER_HEADFUL_POOL_SIZE=1` (off by default). - **AWS WAF Challenge support.** Detect the documented `202` Challenge and `405` CAPTCHA responses from their `x-amzn-waf-action` header, with a conservative two-marker HTML fallback. Silent challenges use a dedicated browser waiter for the domain-matching `aws-waf-token`; interactive CAPTCHA is surfaced as `aws-waf-captcha-required` for a future solver. diff --git a/README.md b/README.md index 4b63ade..c2589cf 100644 --- a/README.md +++ b/README.md @@ -118,11 +118,12 @@ curl -X POST http://localhost:8191/scrape \ -d '{"url":"https://nowsecure.nl","maxTimeout":60000}' ``` -### MCP scraping tool (`/mcp`) +### MCP tools (`/mcp`) -Set `MCP_ENABLED=true` to expose TRAWL's client-independent Streamable HTTP -`scrape_url` tool to any MCP-compatible AI application or agent. It loads known -public URLs; it does not provide web search or ranking. See the +Set `MCP_ENABLED=true` to expose TRAWL's client-independent Streamable HTTP tools +for readable content, HTML, screenshots and browser diagnostics to any +MCP-compatible AI application or agent. They load known public URLs; TRAWL does not +provide web search or ranking. See the [MCP integration guide](./apps/docs/integrations/mcp.md). ### Connect Prowlarr / Jackett diff --git a/apps/docs/getting-started/configuration.md b/apps/docs/getting-started/configuration.md index 7390db3..e95cbdc 100644 --- a/apps/docs/getting-started/configuration.md +++ b/apps/docs/getting-started/configuration.md @@ -13,9 +13,10 @@ All configuration is via environment variables. Copy `.env.example` to `.env` an **Default:** `false` -Enables the Streamable HTTP endpoint at `POST/GET /mcp`. It exposes one read-only -`scrape_url` tool and does not add web search, ranking, or result discovery. Keep the -endpoint on a trusted private network; this first version does not provide authentication. +Enables the Streamable HTTP endpoint at `POST/GET /mcp`. It exposes read-only tools +for readable content, HTML, screenshots and browser diagnostics. It does not add web +search, ranking or result discovery. Keep the endpoint on a trusted private network; +the endpoint does not provide authentication. ```ini MCP_ENABLED=true diff --git a/apps/docs/integrations/mcp.md b/apps/docs/integrations/mcp.md index 998be52..62a0aac 100644 --- a/apps/docs/integrations/mcp.md +++ b/apps/docs/integrations/mcp.md @@ -7,7 +7,7 @@ description: Connect any MCP-compatible AI client to TRAWL's scraping tool. TRAWL has an optional, client-independent Model Context Protocol server. Any AI application or agent that supports remote MCP servers over Streamable HTTP can -connect to it and use the `scrape_url` tool. +connect to it and use its reading, scraping, screenshot and inspection tools. This is a scraper, not a web search engine. The caller must already know the URL. Search discovery and reranking require a separate provider such as SearXNG; TRAWL @@ -40,21 +40,46 @@ The exact configuration syntax belongs to the client. If the client blocks priva network addresses by default, allow the TRAWL hostname or address in that client's network policy. TRAWL does not require or assume any specific AI frontend. -## Tool contract +## Tools -`scrape_url` accepts only: +TRAWL exposes a small set of purpose-specific, read-only tools: -- `url` — required public HTTP(S) URL -- `maxTimeout` — optional positive timeout in milliseconds -- `maxTier` — optional escalation cap from 1 through 4 -- `skipHttp` — optionally skip the plain HTTP tier +| Tool | Use it for | +| --- | --- | +| `read` | Extracting the main page content as Markdown or plain text | +| `scrape` | Reading the original HTML and scrape metadata | +| `screenshot` | Rendering a viewport JPEG for visual or multimodal inspection | +| `inspect` | Browser console, network timing and redirect diagnostics | +| `scrape_url` | Backwards-compatible alias for `scrape` | + +Every tool requires a public HTTP(S) `url` and optionally accepts `maxTimeout` +and `maxTier`. `scrape` also accepts `skipHttp`. Browser-only tools start at +Tier 2 and accept a `maxTier` from 2 through 4. + +`read` defaults to Markdown and a 50,000-character response. Set `format` to +`text` for plain text, or set `maxCharacters` to a value from 1 through 100,000. +Its structured metadata includes the title and, when detected, the byline, +excerpt, site name and language. + +`scrape` returns at most 50,000 characters of page HTML. Its structured output +includes the final URL, status, winning tier, content type, elapsed time, +per-tier attempt history, cache use, truncation and non-sensitive CAPTCHA/proxy +booleans. + +`screenshot` returns an MCP `image` content block containing a base64 JPEG plus +structured scrape metadata. A client and its selected model must support image +tool results to make visual use of it. + +`inspect` returns the bounded diagnostics already collected by TRAWL's browser +tiers. Credentials, query strings and fragments are stripped from network and +redirect URLs. Console messages are page-controlled and can themselves contain +tokens or personal data, so treat this explicit diagnostic tool as sensitive. Private, loopback, link-local, and reserved destinations are rejected, including redirect destinations and browser-loaded subresources. Request headers, bodies, cookies, session IDs, and explicit proxies cannot be supplied by the model. -The response contains at most 50,000 characters of page HTML plus structured -metadata: final URL, status code, tier, content type, elapsed time, and whether the -HTML was truncated. Cookies, proxy details, browser identity, and session data are -never returned. +Cookies, request headers, response bodies captured from background APIs, explicit +proxy details, browser identity and session data are never returned by these MCP +tools.