server.js

// Thin MCP HTTP adapter for web-scrape.
//
// Tools: scrape_website_context (HTTP extract) and browser_fetch_url (Playwright).
// Prefer the CLI (`web-scrape fetch|scrape`) for agents; use this server when an
// MCP client must discover tools over HTTP.
//
//   POST /mcp/v1          JSON-RPC 2.0 (initialize, tools/list, tools/call)
//   GET  /mcp/v1/health   liveness + dependency readiness + effective limits
//   GET  /mcp/v1/ready    readiness probe (dependency self-check)

import http from "node:http";
import dns from "node:dns";
import { fileURLToPath } from "node:url";

import {
  INPUT_SCHEMA,
  OUTPUT_SCHEMA,
  validateInput,
  scrape,
  wasCacheHit,
} from "./schema.js";
import {
  BROWSER_FETCH_TOOL_NAME,
  BROWSER_FETCH_INPUT_SCHEMA,
  BROWSER_FETCH_OUTPUT_SCHEMA,
  validateBrowserFetchInput,
} from "./browser_fetch_schema.js";
import { browserFetchURL } from "./browser_fetch.js";
import { config, limitsSummary } from "./config.js";

const PORT = Number(process.env.PORT || 8091);
const PROTOCOL_VERSION = "2024-11-05";
const SERVER_INFO = { name: "web-scrape", version: "0.1.1" };

// JSON-RPC 2.0 error codes (https://www.jsonrpc.org/specification#error_object)
const codeParseError = -32700;
const codeInvalidRequest = -32600;
const codeMethodNotFound = -32601;
const codeInvalidParams = -32602;
const codeInternalError = -32603;

const TOOLS = [
  {
    name: "scrape_website_context",
    description:
      "Fetch a website and return structured context (title, summary, product " +
      "description, audience signals, positioning claims, pricing/CTA). " +
      "Inputs are bounded; output fields are always present.",
    inputSchema: INPUT_SCHEMA,
    outputSchema: OUTPUT_SCHEMA,
  },
  {
    name: BROWSER_FETCH_TOOL_NAME,
    description:
      "Fetch a public URL in an ephemeral browser and return readable text or a structured status.",
    inputSchema: BROWSER_FETCH_INPUT_SCHEMA,
    outputSchema: BROWSER_FETCH_OUTPUT_SCHEMA,
  },
];

// metrics is a tiny in-process accumulator (#998) so block/timeout/cache rates
// are observable on the health JSON without a metrics backend. Counts only —
// never any scraped content. Per-process (not shared across replicas); resets
// on restart, which is fine for a local/dev sidecar.
const metrics = {
  attempts: 0,
  ok: 0,
  rejects: 0, // non-`ok` outcomes (blocked/captcha/login/rate/robots/timeout/unsupported/error)
  timeouts: 0,
  cacheHits: 0,
  cacheMisses: 0,
  byStatus: Object.create(null), // status -> count, for per-status block rates
};

// recordScrape folds one scrape outcome into the counters.
function recordScrape(status, cached) {
  metrics.attempts += 1;
  metrics.byStatus[status] = (metrics.byStatus[status] || 0) + 1;
  if (status === "ok") metrics.ok += 1;
  else metrics.rejects += 1;
  if (status === "timeout") metrics.timeouts += 1;
  if (cached) metrics.cacheHits += 1;
  else metrics.cacheMisses += 1;
}

// extractionQuality derives content-yield signals from the output WITHOUT
// logging any page text: an `extracted` boolean, a count of non-empty fields,
// and the summary LENGTH (not its contents). This keeps AC 2 — no sensitive
// page content leaks into logs — while still making yield observable (#998).
function extractionQuality(output) {
  const textFields = ["title", "meta_description", "summary", "product_description", "pricing_or_cta"];
  let fields = 0;
  for (const f of textFields) {
    if (typeof output?.[f] === "string" && output[f].trim() !== "") fields += 1;
  }
  for (const a of ["audience_signals", "positioning_claims"]) {
    if (Array.isArray(output?.[a]) && output[a].length > 0) fields += 1;
  }
  const summary = typeof output?.summary === "string" ? output.summary : "";
  return { extracted: output?.status === "ok", fields, summaryLen: summary.length };
}

// logScrape emits one structured, dependency-free line per tools/call so block
// rates, latency, and content yield are observable in production (#1021, #998).
// Status, host, hop count, duration, cache-hit, and extraction-quality COUNTS
// are the signals that matter; the URL host (not the full URL or any content)
// is logged to keep the line small and PII-light (AC 2).
function logScrape(input, output, durationMs) {
  let host = "";
  try {
    host = new URL(input?.url || "").host;
  } catch {
    host = "";
  }
  const hops = Array.isArray(output?.source_pages) ? output.source_pages.length : 0;
  const warnings = Array.isArray(output?.warnings) ? output.warnings.length : 0;
  const status = output?.status || "ok";
  const cached = wasCacheHit(output);
  const quality = extractionQuality(output);
  recordScrape(status, cached);
  const line = {
    evt: "scrape",
    status,
    host,
    hops,
    warnings,
    cached,
    extracted: quality.extracted,
    fields: quality.fields,
    summary_len: quality.summaryLen,
    ms: durationMs,
  };
  console.log(`[web-scrape] ${JSON.stringify(line)}`);
}

function logBrowserFetch(input, output, durationMs) {
  let host = "";
  try {
    host = new URL(input?.url || "").host;
  } catch {
    host = "";
  }
  const status = output?.status || "error";
  recordScrape(status, false);
  const line = {
    evt: "browser_fetch",
    status,
    host,
    warnings: Array.isArray(output?.warnings) ? output.warnings.length : 0,
    truncated: Boolean(output?.truncated),
    content_len: typeof output?.content === "string" ? output.content.length : 0,
    ms: durationMs,
  };
  console.log(`[web-scrape] ${JSON.stringify(line)}`);
}

// readiness performs a CHEAP dependency self-check: the sidecar has no DB; its
// only real dependency is the outbound HTTP/DNS runtime (Node 18+ global fetch
// + AbortController + a DNS resolver). This is a runtime/config self-check, NOT
// an outbound-connectivity guarantee — it deliberately makes no live request so
// it adds no latency and never fail-closes in a restricted network (#998).
function readiness() {
  const deps = {
    fetch: typeof fetch === "function",
    abort_controller: typeof AbortController === "function",
    dns_resolver: typeof dns.lookup === "function",
    config_valid: config.warnings.length === 0,
  };
  // config_valid is informational: a clamped env var is a soft warning, not a
  // reason to refuse traffic. Readiness hinges on the runtime capabilities only.
  const ready = deps.fetch && deps.abort_controller && deps.dns_resolver;
  return { ready, deps };
}

// metricsSnapshot returns a plain copy of the counters for the health JSON.
function metricsSnapshot() {
  return {
    attempts: metrics.attempts,
    ok: metrics.ok,
    rejects: metrics.rejects,
    timeouts: metrics.timeouts,
    cache_hits: metrics.cacheHits,
    cache_misses: metrics.cacheMisses,
    by_status: { ...metrics.byStatus },
  };
}

function sendJSON(res, status, obj) {
  const body = JSON.stringify(obj);
  res.writeHead(status, {
    "Content-Type": "application/json",
    "Content-Length": Buffer.byteLength(body),
  });
  res.end(body);
}

function readBody(req) {
  return new Promise((resolve, reject) => {
    const chunks = [];
    req.on("data", (c) => chunks.push(c));
    req.on("end", () => resolve(Buffer.concat(chunks).toString("utf8")));
    req.on("error", reject);
  });
}

function rpcResult(id, result) {
  return { jsonrpc: "2.0", id: id ?? null, result };
}

function rpcError(id, code, message, data) {
  const err = { code, message };
  if (data !== undefined) err.data = data;
  return { jsonrpc: "2.0", id: id ?? null, error: err };
}

// dispatch handles a single JSON-RPC request object and returns a response
// object, or null for notifications (requests without an id).
async function dispatch(req) {
  if (!req || req.jsonrpc !== "2.0" || typeof req.method !== "string") {
    return rpcError(req?.id, codeInvalidRequest, "invalid JSON-RPC 2.0 request");
  }
  const isNotification = req.id === undefined || req.id === null;

  switch (req.method) {
    case "initialize":
      return rpcResult(req.id, {
        protocolVersion: PROTOCOL_VERSION,
        capabilities: { tools: {} },
        serverInfo: SERVER_INFO,
      });

    case "notifications/initialized":
      return null; // notification; no response

    case "ping":
      return rpcResult(req.id, {});

    case "tools/list":
      return rpcResult(req.id, { tools: TOOLS });

    case "tools/call": {
      const params = req.params || {};
      if (params.name === "scrape_website_context") {
        let input;
        try {
          input = validateInput(params.arguments);
        } catch (e) {
          return rpcError(req.id, codeInvalidParams, String(e?.message || e));
        }
        try {
          const startedAt = Date.now();
          const output = await scrape(input);
          logScrape(input, output, Date.now() - startedAt);
          return rpcResult(req.id, {
            content: [{ type: "text", text: JSON.stringify(output, null, 2) }],
            structuredContent: output,
            isError: false,
          });
        } catch (e) {
          return rpcError(req.id, codeInternalError, String(e?.message || e));
        }
      }
      if (params.name === BROWSER_FETCH_TOOL_NAME) {
        let input;
        try {
          input = validateBrowserFetchInput(params.arguments);
        } catch (e) {
          return rpcError(req.id, codeInvalidParams, String(e?.message || e));
        }
        try {
          const startedAt = Date.now();
          const output = await browserFetchURL(input);
          logBrowserFetch(input, output, Date.now() - startedAt);
          return rpcResult(req.id, {
            content: [{ type: "text", text: JSON.stringify(output, null, 2) }],
            structuredContent: output,
            isError: false,
          });
        } catch (e) {
          return rpcError(req.id, codeInternalError, String(e?.message || e));
        }
      }
      return rpcError(req.id, codeMethodNotFound, `unknown tool: ${params.name}`);
    }

    default:
      if (isNotification) return null;
      return rpcError(req.id, codeMethodNotFound, `method not found: ${req.method}`);
  }
}

const server = http.createServer(async (req, res) => {
  // Liveness + readiness + effective config in one payload. Liveness ("the
  // process is up") always returns 200; the `ready`/`deps` fields report the
  // dependency self-check, and `limits`/`metrics` let an operator confirm the
  // effective config and observe block/timeout/cache rates (#998).
  if (req.method === "GET" && req.url === "/mcp/v1/health") {
    const { ready, deps } = readiness();
    return sendJSON(res, 200, {
      status: "ok",
      protocol: "mcp",
      version: SERVER_INFO.version,
      tools: TOOLS.length,
      ready,
      deps,
      limits: limitsSummary(),
      metrics: metricsSnapshot(),
    });
  }

  // Dedicated readiness probe: 200 when dependencies are ready, 503 otherwise,
  // so an orchestrator (compose/k8s) routes traffic only when the sidecar can
  // actually serve scrapes (AC 1).
  if (req.method === "GET" && req.url === "/mcp/v1/ready") {
    const { ready, deps } = readiness();
    return sendJSON(res, ready ? 200 : 503, {
      status: ready ? "ready" : "not_ready",
      ready,
      deps,
      limits: limitsSummary(),
    });
  }

  if (req.method === "GET" && req.url === "/mcp/v1") {
    return sendJSON(res, 405, {
      error: "MCP endpoint accepts JSON-RPC 2.0 over POST. See /mcp/v1/health for status.",
    });
  }

  if (req.method === "POST" && req.url === "/mcp/v1") {
    let raw;
    try {
      raw = await readBody(req);
    } catch {
      return sendJSON(res, 400, rpcError(null, codeParseError, "failed to read body"));
    }
    let payload;
    try {
      payload = raw ? JSON.parse(raw) : null;
    } catch {
      return sendJSON(res, 200, rpcError(null, codeParseError, "invalid JSON"));
    }
    if (payload === null) {
      return sendJSON(res, 200, rpcError(null, codeInvalidRequest, "empty request body"));
    }

    // JSON-RPC 2.0 batch support (array of requests).
    if (Array.isArray(payload)) {
      const out = [];
      for (const r of payload) {
        const resp = await dispatch(r);
        if (resp !== null) out.push(resp);
      }
      return sendJSON(res, 200, out);
    }

    const resp = await dispatch(payload);
    if (resp === null) {
      res.writeHead(202);
      return res.end();
    }
    return sendJSON(res, 200, resp);
  }

  return sendJSON(res, 404, rpcError(null, codeMethodNotFound, "not found"));
});

// start binds the listener and installs signal handlers. Kept behind a
// main-module guard so importing this file (e.g. from config.test.mjs to assert
// the readiness/metrics shape) never binds a port (#998).
function start() {
  server.listen(PORT, () => {
    console.log(`[web-scrape] MCP JSON-RPC on http://localhost:${PORT}/mcp/v1`);
    console.log(`[web-scrape] ${JSON.stringify({ evt: "config", limits: limitsSummary() })}`);
    for (const w of config.warnings) {
      console.warn(`[web-scrape] config warning: ${w}`);
    }
  });

  for (const sig of ["SIGINT", "SIGTERM"]) {
    process.on(sig, () => {
      server.close(() => process.exit(0));
    });
  }
  return server;
}

// Run as a server only when invoked directly (node server.js / npm start), not
// when imported by a test.
const invokedDirectly =
  process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1];
if (invokedDirectly) start();

// Exported for validation/unit tests (and embedding) — the dependency self-check
// and metrics snapshot shape are asserted without booting a listener.
export { server, start, readiness, metricsSnapshot, recordScrape, extractionQuality };