diff --git a/worker/src/index.ts b/worker/src/index.ts index 4872920..5532a6a 100644 --- a/worker/src/index.ts +++ b/worker/src/index.ts @@ -38,7 +38,7 @@ import { quickCatchPage, } from "./stripe"; import { listManagedConnections } from "./connections"; -import { track } from "./metrics"; +import { track, trackView } from "./metrics"; import { resolveTools, executeTool, cacheKey, normalizeUrl, writeCache, executeCapturedTool, listCapturedTools } from "./engine"; type Bindings = { @@ -69,6 +69,29 @@ app.use("*", cors({ maxAge: 86400, })); +// Page-view instrumentation: record arrival page + traffic source for every GET +// (no-ops on assets/API inside trackView). Turns the blind "25k visitors with 0 +// signups" number into top-landing-pages + top-referrers, read from the land:/ +// ref: KV counters or GET /api/v1/admin/metrics. Fire-and-forget, never blocks. +app.use("*", async (c, next) => { + if (c.req.method === "GET") { + try { + const u = new URL(c.req.url); + trackView( + c.env, + c.executionCtx, + u.pathname, + c.req.header("referer") || c.req.header("Referer"), + u, + c.req.header("user-agent") || "" + ); + } catch { + /* never break a request */ + } + } + await next(); +}); + // --------------------- helpers --------------------- // cacheKey / normalizeUrl / writeCache (and the extraction/execution cascade // resolveTools/executeTool) are imported from ./engine — one source of truth diff --git a/worker/src/metrics.ts b/worker/src/metrics.ts index e5c49ec..b8a2131 100644 --- a/worker/src/metrics.ts +++ b/worker/src/metrics.ts @@ -54,6 +54,128 @@ export function track( } } +// --------------------------------------------------------------------------- +// Landing-page + traffic-source dimensions. +// +// The event funnel above answers "what did people DO"; these answer "where did +// they LAND and where did they come FROM" — the questions you can't answer when +// a traffic spike shows up with zero signups. Separate key prefixes so the +// funnel rollup stays clean: +// land:: -> count (arrival page) +// ref:: -> count (referrer / utm_source, or "bot") +const DIM_TTL_DAYS = 60; + +function bumpDaily( + env: { USAGE: KVNamespace }, + ctx: { waitUntil(p: Promise): void }, + key: string +): void { + try { + ctx.waitUntil( + (async () => { + const cur = await env.USAGE.get(key); + const n = cur ? parseInt(cur, 10) : 0; + await env.USAGE.put(key, String(n + 1), { + expirationTtl: DIM_TTL_DAYS * 86400, + }); + })() + ); + } catch { + /* never break a request */ + } +} + +// Obvious crawlers/automation — so the source rollup separates real humans from +// the bot share of a "25k visitors" number. +function isBot(ua: string): boolean { + return /bot\b|crawl|spider|slurp|bingpreview|facebookexternalhit|headless|phantom|python-requests|curl\/|wget|axios|go-http|java\/|okhttp|scrapy|semrush|ahrefs|dataforseo|mj12|dotbot|gptbot|claudebot|ccbot|amazonbot|bytespider|google-extended|petalbot|applebot/i.test( + ua + ); +} + +// Coarse landing-page label for a request path. Returns null for assets / API / +// machine endpoints (don't pollute the page funnel). +export function pageBucket(path: string): string | null { + if (path === "/") return "home"; + if (/^\/(api|static|assets|_)\//.test(path)) return null; + if (/\.(js|css|png|jpe?g|gif|svg|ico|txt|xml|json|webp|woff2?|map|wasm)$/i.test(path)) return null; + if (["/favicon.ico", "/robots.txt", "/sitemap.xml", "/llms.txt"].includes(path)) return null; + if (path.startsWith("/u/")) return "u"; + if (path.startsWith("/drops")) return "drops"; + if (path.startsWith("/guides")) return "guides"; + if (path.startsWith("/blog")) return "blog"; + if (path.startsWith("/directory")) return "directory"; + if (path.startsWith("/mcp/leaderboard")) return "leaderboard"; + if (path.startsWith("/mcp/grade")) return "grade"; + if (path.startsWith("/mcp")) return "mcp"; + if (path.startsWith("/pricing")) return "pricing"; + if (path.startsWith("/tools")) return "tools"; + if (path.startsWith("/quickcatch")) return "quickcatch"; + if (path.startsWith("/connect")) return "connect"; + if (path.startsWith("/webmcp")) return "webmcp"; + if (path.startsWith("/dashboard")) return "dashboard"; + return "other"; +} + +// Traffic source from utm_source/source override, then Referer host, then bot +// UA, else "direct". +export function sourceBucket( + referer: string | undefined, + url: URL, + ua: string +): string { + const tagged = (url.searchParams.get("utm_source") || url.searchParams.get("source") || "") + .toLowerCase() + .replace(/[^a-z0-9_]/g, "") + .slice(0, 24); + if (tagged) return tagged.startsWith("ext") ? "extension" : tagged; + if (ua && isBot(ua)) return "bot"; + if (!referer) return "direct"; + let host = ""; + try { + host = new URL(referer).hostname.replace(/^www\./, "").toLowerCase(); + } catch { + return "direct"; + } + if (!host) return "direct"; + if (host.endsWith("wmcp.sh")) return "internal"; + const map: [RegExp, string][] = [ + [/google\./, "google"], [/bing\./, "bing"], [/duckduckgo\./, "duckduckgo"], + [/yandex\./, "yandex"], [/reddit\.|redd\.it/, "reddit"], + [/twitter\.com|x\.com|t\.co$/, "x"], [/tiktok\./, "tiktok"], + [/youtube\.|youtu\.be/, "youtube"], [/facebook\.|fb\./, "facebook"], + [/instagram\./, "instagram"], [/linkedin\.|lnkd\.in/, "linkedin"], + [/github\./, "github"], [/ycombinator|hckrnews/, "hackernews"], + [/chatgpt\.|openai\./, "chatgpt"], [/perplexity\./, "perplexity"], + [/claude\.|anthropic\./, "claude"], [/producthunt\./, "producthunt"], + [/discord\.|discordapp/, "discord"], [/t\.me|telegram/, "telegram"], + ]; + for (const [re, name] of map) if (re.test(host)) return name; + // Unknown referrer: keep the eTLD+1 so we can actually see where it came from. + return host.split(".").slice(-2).join(".").slice(0, 24); +} + +// Fire-and-forget: record arrival page + source for a page view. No-ops on +// assets/API. Call from a GET middleware. +export function trackView( + env: { USAGE: KVNamespace }, + ctx: { waitUntil(p: Promise): void }, + path: string, + referer: string | undefined, + url: URL, + ua: string +): void { + try { + const bucket = pageBucket(path); + if (!bucket) return; + const day = new Date().toISOString().slice(0, 10); + bumpDaily(env, ctx, `land:${bucket}:${day}`); + bumpDaily(env, ctx, `ref:${sourceBucket(referer, url, ua)}:${day}`); + } catch { + /* never break a request */ + } +} + type MetricsEnv = { USAGE: KVNamespace; CACHE?: KVNamespace; @@ -129,5 +251,34 @@ export async function getMetrics(c: Context<{ Bindings: MetricsEnv }>) { } const assets = { cached_urls, directory_sites: sites.size, graded_servers, grade_queue }; - return c.json({ events: EVENTS, totals, funnel, byDay, assets }); + // Landing-page + traffic-source rollups (where the visitors land / come from). + const landing: Record = {}; + const sources: Record = {}; + for (const [prefix, acc] of [["land:", landing], ["ref:", sources]] as const) { + try { + const l = await c.env.USAGE.list({ prefix, limit: 1000 }); + await Promise.all( + l.keys.map(async (k) => { + const parts = k.name.split(":"); // : (bucket may hold a dot, not a colon) + const day = parts[parts.length - 1]; + const bucket = parts.slice(1, -1).join(":"); + if (!bucket || !day) return; + const raw = await c.env.USAGE.get(k.name); + acc[bucket] = (acc[bucket] || 0) + (raw ? parseInt(raw, 10) : 0); + }) + ); + } catch {} + } + const topN = (o: Record, n = 25) => + Object.fromEntries(Object.entries(o).sort((a, b) => b[1] - a[1]).slice(0, n)); + + return c.json({ + events: EVENTS, + totals, + funnel, + byDay, + assets, + landing: topN(landing), + sources: topN(sources), + }); } diff --git a/worker/test/metrics_dims.test.ts b/worker/test/metrics_dims.test.ts new file mode 100644 index 0000000..132b1d6 --- /dev/null +++ b/worker/test/metrics_dims.test.ts @@ -0,0 +1,58 @@ +import { describe, it, expect } from "vitest"; +import { pageBucket, sourceBucket } from "../src/metrics"; + +const url = (s: string) => new URL(s); + +describe("pageBucket", () => { + it("labels the key landing pages", () => { + expect(pageBucket("/")).toBe("home"); + expect(pageBucket("/drops")).toBe("drops"); + expect(pageBucket("/drops/prismatic-evolutions-restock")).toBe("drops"); + expect(pageBucket("/guides/best-booster-boxes")).toBe("guides"); + expect(pageBucket("/mcp/leaderboard")).toBe("leaderboard"); + expect(pageBucket("/mcp/grade")).toBe("grade"); + expect(pageBucket("/pricing")).toBe("pricing"); + expect(pageBucket("/u/aHR0cHM6Ly9leA")).toBe("u"); + expect(pageBucket("/quickcatch")).toBe("quickcatch"); + expect(pageBucket("/some/unknown")).toBe("other"); + }); + + it("ignores assets, API and machine endpoints", () => { + expect(pageBucket("/api/v1/stats/public")).toBeNull(); + expect(pageBucket("/static/app.js")).toBeNull(); + expect(pageBucket("/x/logo.png")).toBeNull(); + expect(pageBucket("/robots.txt")).toBeNull(); + expect(pageBucket("/sitemap.xml")).toBeNull(); + expect(pageBucket("/favicon.ico")).toBeNull(); + }); +}); + +describe("sourceBucket", () => { + const u = url("https://wmcp.sh/drops"); + + it("prefers an explicit utm_source/source tag", () => { + expect(sourceBucket(undefined, url("https://wmcp.sh/?utm_source=reddit"), "")).toBe("reddit"); + expect(sourceBucket(undefined, url("https://wmcp.sh/?source=ext_paywall"), "")).toBe("extension"); + }); + + it("classifies known referrers", () => { + expect(sourceBucket("https://www.google.com/search?q=x", u, "")).toBe("google"); + expect(sourceBucket("https://www.reddit.com/r/PTCGL/", u, "")).toBe("reddit"); + expect(sourceBucket("https://t.co/abc", u, "")).toBe("x"); + expect(sourceBucket("https://news.ycombinator.com/", u, "")).toBe("hackernews"); + }); + + it("separates bots from humans via user-agent", () => { + expect(sourceBucket(undefined, u, "Mozilla/5.0 (compatible; Googlebot/2.1)")).toBe("bot"); + expect(sourceBucket(undefined, u, "GPTBot/1.0")).toBe("bot"); + expect(sourceBucket(undefined, u, "Mozilla/5.0 (Macintosh) Safari/605")).toBe("direct"); + }); + + it("keeps the eTLD+1 for unknown referrers", () => { + expect(sourceBucket("https://blog.someforum.io/thread", u, "")).toBe("someforum.io"); + }); + + it("marks internal navigation", () => { + expect(sourceBucket("https://wmcp.sh/guides", u, "")).toBe("internal"); + }); +});