Funnel visibility: landing-page + traffic-source metrics

worker/src/index.ts

@@ -38,7 +38,7 @@ import {
   quickCatchPage,
 } from "./stripe";
 import { listManagedConnections } from "./connections";
-import { track } from "./metrics";
+import { track, trackView } from "./metrics";
 import { resolveTools, executeTool, cacheKey, normalizeUrl, writeCache, executeCapturedTool, listCapturedTools } from "./engine";
 
 type Bindings = {
@@ -69,6 +69,29 @@ app.use("*", cors({
   maxAge: 86400,
 }));
 
+// Page-view instrumentation: record arrival page + traffic source for every GET
+// (no-ops on assets/API inside trackView). Turns the blind "25k visitors with 0
+// signups" number into top-landing-pages + top-referrers, read from the land:/
+// ref: KV counters or GET /api/v1/admin/metrics. Fire-and-forget, never blocks.
+app.use("*", async (c, next) => {
+  if (c.req.method === "GET") {
+    try {
+      const u = new URL(c.req.url);
+      trackView(
+        c.env,
+        c.executionCtx,
+        u.pathname,
+        c.req.header("referer") || c.req.header("Referer"),
+        u,
+        c.req.header("user-agent") || ""
+      );
+    } catch {
+      /* never break a request */
+    }
+  }
+  await next();
+});
+
 // --------------------- helpers ---------------------
 // cacheKey / normalizeUrl / writeCache (and the extraction/execution cascade
 // resolveTools/executeTool) are imported from ./engine — one source of truth

worker/src/metrics.ts

@@ -54,6 +54,128 @@ export function track(
   }
 }
 
+// ---------------------------------------------------------------------------
+// Landing-page + traffic-source dimensions.
+//
+// The event funnel above answers "what did people DO"; these answer "where did
+// they LAND and where did they come FROM" — the questions you can't answer when
+// a traffic spike shows up with zero signups. Separate key prefixes so the
+// funnel rollup stays clean:
+//   land:<bucket>:<YYYY-MM-DD> -> count   (arrival page)
+//   ref:<source>:<YYYY-MM-DD>  -> count   (referrer / utm_source, or "bot")
+const DIM_TTL_DAYS = 60;
+
+function bumpDaily(
+  env: { USAGE: KVNamespace },
+  ctx: { waitUntil(p: Promise<unknown>): void },
+  key: string
+): void {
+  try {
+    ctx.waitUntil(
+      (async () => {
+        const cur = await env.USAGE.get(key);
+        const n = cur ? parseInt(cur, 10) : 0;
+        await env.USAGE.put(key, String(n + 1), {
+          expirationTtl: DIM_TTL_DAYS * 86400,
+        });
+      })()
+    );
+  } catch {
+    /* never break a request */
+  }
+}
+
+// Obvious crawlers/automation — so the source rollup separates real humans from
+// the bot share of a "25k visitors" number.
+function isBot(ua: string): boolean {
+  return /bot\b|crawl|spider|slurp|bingpreview|facebookexternalhit|headless|phantom|python-requests|curl\/|wget|axios|go-http|java\/|okhttp|scrapy|semrush|ahrefs|dataforseo|mj12|dotbot|gptbot|claudebot|ccbot|amazonbot|bytespider|google-extended|petalbot|applebot/i.test(
+    ua
+  );
+}
+
+// Coarse landing-page label for a request path. Returns null for assets / API /
+// machine endpoints (don't pollute the page funnel).
+export function pageBucket(path: string): string | null {
+  if (path === "/") return "home";
+  if (/^\/(api|static|assets|_)\//.test(path)) return null;
+  if (/\.(js|css|png|jpe?g|gif|svg|ico|txt|xml|json|webp|woff2?|map|wasm)$/i.test(path)) return null;
+  if (["/favicon.ico", "/robots.txt", "/sitemap.xml", "/llms.txt"].includes(path)) return null;
+  if (path.startsWith("/u/")) return "u";
+  if (path.startsWith("/drops")) return "drops";
+  if (path.startsWith("/guides")) return "guides";
+  if (path.startsWith("/blog")) return "blog";
+  if (path.startsWith("/directory")) return "directory";
+  if (path.startsWith("/mcp/leaderboard")) return "leaderboard";
+  if (path.startsWith("/mcp/grade")) return "grade";
+  if (path.startsWith("/mcp")) return "mcp";
+  if (path.startsWith("/pricing")) return "pricing";
+  if (path.startsWith("/tools")) return "tools";
+  if (path.startsWith("/quickcatch")) return "quickcatch";
+  if (path.startsWith("/connect")) return "connect";
+  if (path.startsWith("/webmcp")) return "webmcp";
+  if (path.startsWith("/dashboard")) return "dashboard";
+  return "other";
+}
+
+// Traffic source from utm_source/source override, then Referer host, then bot
+// UA, else "direct".
+export function sourceBucket(
+  referer: string | undefined,
+  url: URL,
+  ua: string
+): string {
+  const tagged = (url.searchParams.get("utm_source") || url.searchParams.get("source") || "")
+    .toLowerCase()
+    .replace(/[^a-z0-9_]/g, "")
+    .slice(0, 24);
+  if (tagged) return tagged.startsWith("ext") ? "extension" : tagged;
+  if (ua && isBot(ua)) return "bot";
+  if (!referer) return "direct";
+  let host = "";
+  try {
+    host = new URL(referer).hostname.replace(/^www\./, "").toLowerCase();
+  } catch {
+    return "direct";
+  }
+  if (!host) return "direct";
+  if (host.endsWith("wmcp.sh")) return "internal";
+  const map: [RegExp, string][] = [
+    [/google\./, "google"], [/bing\./, "bing"], [/duckduckgo\./, "duckduckgo"],
+    [/yandex\./, "yandex"], [/reddit\.|redd\.it/, "reddit"],
+    [/twitter\.com|x\.com|t\.co$/, "x"], [/tiktok\./, "tiktok"],
+    [/youtube\.|youtu\.be/, "youtube"], [/facebook\.|fb\./, "facebook"],
+    [/instagram\./, "instagram"], [/linkedin\.|lnkd\.in/, "linkedin"],
+    [/github\./, "github"], [/ycombinator|hckrnews/, "hackernews"],
+    [/chatgpt\.|openai\./, "chatgpt"], [/perplexity\./, "perplexity"],
+    [/claude\.|anthropic\./, "claude"], [/producthunt\./, "producthunt"],
+    [/discord\.|discordapp/, "discord"], [/t\.me|telegram/, "telegram"],
+  ];
+  for (const [re, name] of map) if (re.test(host)) return name;
+  // Unknown referrer: keep the eTLD+1 so we can actually see where it came from.
+  return host.split(".").slice(-2).join(".").slice(0, 24);
+}
+
+// Fire-and-forget: record arrival page + source for a page view. No-ops on
+// assets/API. Call from a GET middleware.
+export function trackView(
+  env: { USAGE: KVNamespace },
+  ctx: { waitUntil(p: Promise<unknown>): void },
+  path: string,
+  referer: string | undefined,
+  url: URL,
+  ua: string
+): void {
+  try {
+    const bucket = pageBucket(path);
+    if (!bucket) return;
+    const day = new Date().toISOString().slice(0, 10);
+    bumpDaily(env, ctx, `land:${bucket}:${day}`);
+    bumpDaily(env, ctx, `ref:${sourceBucket(referer, url, ua)}:${day}`);
+  } catch {
+    /* never break a request */
+  }
+}
+
 type MetricsEnv = {
   USAGE: KVNamespace;
   CACHE?: KVNamespace;
@@ -129,5 +251,34 @@ export async function getMetrics(c: Context<{ Bindings: MetricsEnv }>) {
   }
   const assets = { cached_urls, directory_sites: sites.size, graded_servers, grade_queue };
 
-  return c.json({ events: EVENTS, totals, funnel, byDay, assets });
+  // Landing-page + traffic-source rollups (where the visitors land / come from).
+  const landing: Record<string, number> = {};
+  const sources: Record<string, number> = {};
+  for (const [prefix, acc] of [["land:", landing], ["ref:", sources]] as const) {
+    try {
+      const l = await c.env.USAGE.list({ prefix, limit: 1000 });
+      await Promise.all(
+        l.keys.map(async (k) => {
+          const parts = k.name.split(":"); // <prefix><bucket>:<day> (bucket may hold a dot, not a colon)
+          const day = parts[parts.length - 1];
+          const bucket = parts.slice(1, -1).join(":");
+          if (!bucket || !day) return;
+          const raw = await c.env.USAGE.get(k.name);
+          acc[bucket] = (acc[bucket] || 0) + (raw ? parseInt(raw, 10) : 0);
+        })
+      );
+    } catch {}
+  }
+  const topN = (o: Record<string, number>, n = 25) =>
+    Object.fromEntries(Object.entries(o).sort((a, b) => b[1] - a[1]).slice(0, n));
+
+  return c.json({
+    events: EVENTS,
+    totals,
+    funnel,
+    byDay,
+    assets,
+    landing: topN(landing),
+    sources: topN(sources),
+  });
 }

worker/test/metrics_dims.test.ts

@@ -0,0 +1,58 @@
+import { describe, it, expect } from "vitest";
+import { pageBucket, sourceBucket } from "../src/metrics";
+
+const url = (s: string) => new URL(s);
+
+describe("pageBucket", () => {
+  it("labels the key landing pages", () => {
+    expect(pageBucket("/")).toBe("home");
+    expect(pageBucket("/drops")).toBe("drops");
+    expect(pageBucket("/drops/prismatic-evolutions-restock")).toBe("drops");
+    expect(pageBucket("/guides/best-booster-boxes")).toBe("guides");
+    expect(pageBucket("/mcp/leaderboard")).toBe("leaderboard");
+    expect(pageBucket("/mcp/grade")).toBe("grade");
+    expect(pageBucket("/pricing")).toBe("pricing");
+    expect(pageBucket("/u/aHR0cHM6Ly9leA")).toBe("u");
+    expect(pageBucket("/quickcatch")).toBe("quickcatch");
+    expect(pageBucket("/some/unknown")).toBe("other");
+  });
+
+  it("ignores assets, API and machine endpoints", () => {
+    expect(pageBucket("/api/v1/stats/public")).toBeNull();
+    expect(pageBucket("/static/app.js")).toBeNull();
+    expect(pageBucket("/x/logo.png")).toBeNull();
+    expect(pageBucket("/robots.txt")).toBeNull();
+    expect(pageBucket("/sitemap.xml")).toBeNull();
+    expect(pageBucket("/favicon.ico")).toBeNull();
+  });
+});
+
+describe("sourceBucket", () => {
+  const u = url("https://wmcp.sh/drops");
+
+  it("prefers an explicit utm_source/source tag", () => {
+    expect(sourceBucket(undefined, url("https://wmcp.sh/?utm_source=reddit"), "")).toBe("reddit");
+    expect(sourceBucket(undefined, url("https://wmcp.sh/?source=ext_paywall"), "")).toBe("extension");
+  });
+
+  it("classifies known referrers", () => {
+    expect(sourceBucket("https://www.google.com/search?q=x", u, "")).toBe("google");
+    expect(sourceBucket("https://www.reddit.com/r/PTCGL/", u, "")).toBe("reddit");
+    expect(sourceBucket("https://t.co/abc", u, "")).toBe("x");
+    expect(sourceBucket("https://news.ycombinator.com/", u, "")).toBe("hackernews");
+  });
+
+  it("separates bots from humans via user-agent", () => {
+    expect(sourceBucket(undefined, u, "Mozilla/5.0 (compatible; Googlebot/2.1)")).toBe("bot");
+    expect(sourceBucket(undefined, u, "GPTBot/1.0")).toBe("bot");
+    expect(sourceBucket(undefined, u, "Mozilla/5.0 (Macintosh) Safari/605")).toBe("direct");
+  });
+
+  it("keeps the eTLD+1 for unknown referrers", () => {
+    expect(sourceBucket("https://blog.someforum.io/thread", u, "")).toBe("someforum.io");
+  });
+
+  it("marks internal navigation", () => {
+    expect(sourceBucket("https://wmcp.sh/guides", u, "")).toBe("internal");
+  });
+});