From e9cd8f64bae94cf855b555c8335da7dfbedd6da3 Mon Sep 17 00:00:00 2001 From: Alaeddin <15094821+BSalaeddin@users.noreply.github.com> Date: Sat, 12 Sep 2026 03:46:09 +0100 Subject: [PATCH] fix(seo): canonical store sitemap with lastmod behind one indexability policy --- apps/caramel-app/e2e/seo-a11y.spec.ts | 51 ++++ apps/caramel-app/e2e/seo-regression.spec.ts | 30 ++- .../migration.sql | 19 ++ .../app/(marketing)/coupons/[store]/page.tsx | 35 ++- .../src/app/(marketing)/sources/page.tsx | 23 +- apps/caramel-app/src/app/sitemap.ts | 68 ++++-- .../src/lib/catalog/applyCatalogRows.ts | 17 +- apps/caramel-app/src/lib/couponsDb.ts | 15 ++ apps/caramel-app/src/lib/couponsRepo.ts | 52 ++++- apps/caramel-app/src/lib/seo/sitemapStores.ts | 82 +++++++ .../src/lib/seo/storeIndexability.ts | 66 ++++++ .../tests/integration/coupons-read.itest.ts | 40 ++++ .../tests/integration/ingest-catalog.itest.ts | 44 ++++ .../unit/applyCatalogRows-site-case.test.ts | 95 ++++++++ .../tests/unit/couponsRepo.test.ts | 113 ++++++++- apps/caramel-app/tests/unit/sitemap.test.ts | 221 ++++++++++++++++++ .../tests/unit/sitemapStores.test.ts | 172 ++++++++++++++ .../tests/unit/sources-page.test.ts | 75 ++++++ .../tests/unit/storeIndexability.test.ts | 98 ++++++++ 19 files changed, 1277 insertions(+), 39 deletions(-) create mode 100644 apps/caramel-app/prisma/migrations/20260912120000_lowercase_coupon_sites/migration.sql create mode 100644 apps/caramel-app/src/lib/seo/sitemapStores.ts create mode 100644 apps/caramel-app/src/lib/seo/storeIndexability.ts create mode 100644 apps/caramel-app/tests/unit/applyCatalogRows-site-case.test.ts create mode 100644 apps/caramel-app/tests/unit/sitemap.test.ts create mode 100644 apps/caramel-app/tests/unit/sitemapStores.test.ts create mode 100644 apps/caramel-app/tests/unit/sources-page.test.ts create mode 100644 apps/caramel-app/tests/unit/storeIndexability.test.ts diff --git a/apps/caramel-app/e2e/seo-a11y.spec.ts b/apps/caramel-app/e2e/seo-a11y.spec.ts index 3def9a97..8470f171 100644 --- a/apps/caramel-app/e2e/seo-a11y.spec.ts +++ b/apps/caramel-app/e2e/seo-a11y.spec.ts @@ -1,4 +1,9 @@ import { expect, test } from '@playwright/test' +// The app's own slug→registrable-domain rule (tldts / Public Suffix List). +// Relative import on purpose: e2e collection runs in BOTH contexts (hermetic +// and deployed, no generated prisma client) and this module is pure — no +// `@/` alias, no prisma, no env. +import { resolveStoreDomain } from '../src/lib/storeDomain' test.describe('SEO & Accessibility Basics', () => { test('home page has correct title', async ({ page }) => { @@ -152,6 +157,52 @@ test.describe('Coupon pages — crawler-visible SEO', () => { const robots = await page.request.get('/robots.txt') expect(robots.status()).toBe(200) }) + + // Sitemap ↔ canonical agreement (GSC audit 2026-09-11: 84 subdomain slugs, + // 2 mixed-case slugs and 1 non-domain were listed whose own page + // canonicalized elsewhere, and 4,289 of 4,317 sitemap URLs were unknown to + // Google). Every store must BE the canonical the page emits — i.e. + // its own resolveStoreDomain, lowercase — and nothing may repeat. Holds + // on any non-empty catalog, so it stays ungated (two-context rule). + test('every store in sitemap.xml is its own canonical base (lowercase, PSL-resolved) and no repeats', async ({ + page, + }) => { + const res = await page.request.get('/sitemap.xml') + expect(res.status()).toBe(200) + const xml = await res.text() + + const locs = Array.from(xml.matchAll(/([^<]+)<\/loc>/g)).map( + m => m[1]!, + ) + expect(locs.length).toBeGreaterThan(0) + expect(new Set(locs).size, 'duplicate in sitemap.xml').toBe( + locs.length, + ) + + const storeSlugs = locs + .map(loc => /\/coupons\/([^/?#]+)$/.exec(loc)?.[1]) + .filter((slug): slug is string => Boolean(slug)) + .map(slug => decodeURIComponent(slug)) + // The catalog is never legitimately empty in either context. + expect(storeSlugs.length).toBeGreaterThan(0) + + const offenders = storeSlugs.filter( + slug => + slug !== slug.toLowerCase() || + resolveStoreDomain(slug) !== slug, + ) + expect( + offenders, + `store s that are not their own canonical base: ${offenders.slice(0, 20).join(', ')}`, + ).toEqual([]) + }) + + test('/support is listed in sitemap.xml', async ({ baseURL, page }) => { + const res = await page.request.get('/sitemap.xml') + expect(res.status()).toBe(200) + const origin = (baseURL ?? '').replace(/\/+$/, '') + expect(await res.text()).toContain(`${origin}/support`) + }) }) test.describe('Responsive - Mobile Viewport', () => { diff --git a/apps/caramel-app/e2e/seo-regression.spec.ts b/apps/caramel-app/e2e/seo-regression.spec.ts index ac55dffa..36193c96 100644 --- a/apps/caramel-app/e2e/seo-regression.spec.ts +++ b/apps/caramel-app/e2e/seo-regression.spec.ts @@ -28,6 +28,7 @@ import { expect, test } from '@playwright/test' // /pricing 1453 / 1453 -> min 1000 // /sources 499 / 499 -> min 350 // /privacy 2780 / 2784 -> min 1900 +// /support 547 (prod, 2026-09-11) -> min 380 // (/coupons is thin on purpose: the card grid is a client fetch; its server // HTML carries the shell copy + sidebar. If a route legitimately gains or // loses big copy, re-measure with the snippet in the PR that added this file @@ -39,6 +40,7 @@ const ROUTES: ReadonlyArray<{ path: string; minVisibleChars: number }> = [ { path: '/pricing', minVisibleChars: 1000 }, { path: '/sources', minVisibleChars: 350 }, { path: '/privacy', minVisibleChars: 1900 }, + { path: '/support', minVisibleChars: 380 }, ] // Same production-origin set as src/app/robots.ts (and next.config.mjs's @@ -280,14 +282,40 @@ test.describe('SEO regression gate (raw server HTML)', () => { (xml.match(/<\/loc>/g) ?? []).length, ) - // The 6 static marketing routes, emitted against the deployment's + // The static marketing routes, emitted against the deployment's // own origin (sitemap.ts builds each from BASE_URL, which // matches the origin this suite targets in all CI contexts). + // + // /sources is the one conditional entry: sitemap.ts lists it only + // when there is ≥1 ACTIVE source (otherwise the page is an empty + // shell that noindexes itself), so it is asserted against the SAME + // read the sitemap uses — /api/sources — rather than assumed. The + // hermetic seed has 2 ACTIVE sources; the deployed site may have 0. const origin = stripTrailingSlash(baseURL ?? '') for (const { path: routePath } of ROUTES) { + if (routePath === '/sources') continue const loc = `${origin}${routePath}` expect(xml, `sitemap.xml missing ${loc}`).toContain(loc) } + + const sourcesRes = await page.request.get('/api/sources') + expect(sourcesRes.ok()).toBe(true) + const sourcesBody = (await sourcesRes.json()) as { data?: unknown } + const activeSources = Array.isArray(sourcesBody.data) + ? sourcesBody.data.length + : 0 + const sourcesLoc = `${origin}/sources` + if (activeSources > 0) { + expect( + xml, + `${activeSources} ACTIVE source(s) but sitemap.xml omits ${sourcesLoc}`, + ).toContain(sourcesLoc) + } else { + expect( + xml, + `0 ACTIVE sources but sitemap.xml lists ${sourcesLoc}`, + ).not.toContain(sourcesLoc) + } }) test('llms.txt is served for answer engines', async ({ page }) => { diff --git a/apps/caramel-app/prisma/migrations/20260912120000_lowercase_coupon_sites/migration.sql b/apps/caramel-app/prisma/migrations/20260912120000_lowercase_coupon_sites/migration.sql new file mode 100644 index 00000000..48295c17 --- /dev/null +++ b/apps/caramel-app/prisma/migrations/20260912120000_lowercase_coupon_sites/migration.sql @@ -0,0 +1,19 @@ +-- Data-only migration: NO schema change (prisma migrate diff against the +-- schema stays empty, so the schema-drift job is unaffected). +-- +-- Domains are case-insensitive, but `coupons.site` was stored verbatim from +-- the producer and the store-page reads match `site = $base OR site LIKE +-- '%.' || $base` case-SENSITIVELY on a lowercased $base (resolveStoreDomain +-- lowercases). Measured on prod 2026-09-11: 2 sites carried uppercase +-- (`Brooklinen.com`, `eNasco.com`) — the eNasco rows were unreachable from +-- ANY store page (/coupons/enasco.com and /coupons/eNasco.com both rendered +-- the empty state with noindex) while the sitemap listed eNasco.com anyway. +-- +-- From this migration on, applyCatalogRows lowercases `site` on every write, +-- so this backfill is the one-time catch-up. Idempotent: the WHERE touches +-- only rows that actually differ, so a re-run is a no-op. `updated_at` is +-- deliberately NOT bumped — the ingest only-if-newer rule keys on it, and a +-- bump would freeze these rows against the producer's next push. +UPDATE "public"."coupons" +SET site = lower(site) +WHERE site IS NOT NULL AND site <> lower(site); diff --git a/apps/caramel-app/src/app/(marketing)/coupons/[store]/page.tsx b/apps/caramel-app/src/app/(marketing)/coupons/[store]/page.tsx index c4927fe9..ac720945 100644 --- a/apps/caramel-app/src/app/(marketing)/coupons/[store]/page.tsx +++ b/apps/caramel-app/src/app/(marketing)/coupons/[store]/page.tsx @@ -5,6 +5,7 @@ import { attachSignals } from '@/lib/couponSignals' import { listStoreCoupons } from '@/lib/couponsRepo' import { BASE_URL } from '@/lib/env.client' import { jsonLdString } from '@/lib/jsonLd' +import { evaluateStorePageIndexability } from '@/lib/seo/storeIndexability' import { resolveStoreDomain } from '@/lib/storeDomain' import type { Coupon } from '@/types/coupon' import type { Metadata } from 'next' @@ -67,11 +68,27 @@ export async function generateMetadata({ const { store } = await Promise.resolve(params) const storeParam = typeof store === 'string' ? safeDecode(store) : '' const base = getBaseDomain(storeParam) - if (!storeParam || !base) { + // ONE indexability policy, shared with the sitemap (app/sitemap.ts via + // src/lib/seo/sitemapStores.ts): a slug naming no registrable store, or a + // store with zero visible coupons, is `noindex, follow`, and the sitemap + // omits exactly those pages. fetchStoreCoupons short-circuits (no catalog + // read) when `base` is empty, and cache() shares the read with the body. + const { total } = await fetchStoreCoupons(storeParam) + const verdict = evaluateStorePageIndexability({ + base, + visibleCouponCount: total, + }) + // `follow` stays on in every noindex case: the links off the page (popular + // stores, header, footer) are still worth crawling. + const robots = verdict.indexable + ? undefined + : ({ index: false, follow: true } as const) + + if (verdict.reason === 'not-a-store') { /* A slug that resolves to no registrable domain is not a store at all, * and this route still answers 200 for it (the body renders the honest * empty state rather than 404ing). That is the soft-404 bloat the - * zero-coupon rule below exists to keep out of the index — only more so, + * zero-coupon rule exists to keep out of the index — only more so, * because there is no store here to have coupons in the first place. * * It only became reachable when getBaseDomain moved to the Public Suffix @@ -79,12 +96,11 @@ export async function generateMetadata({ * this branch was effectively dead and inherited no robots directive. * Caught by e2e/seo-a11y.spec.ts, which asks for /coupons/…-zz.example — * a slug the PSL correctly refuses, since `.example` is reserved and - * cannot be registered. `follow` stays on for the same reason it does - * below: the links off the page are still worth crawling. */ + * cannot be registered. */ return { title: 'Coupons | Caramel', description: 'Find coupons and promo codes on Caramel.', - robots: { index: false, follow: true }, + robots, } } // Declaring `openGraph` below REPLACES the root layout's object wholesale @@ -100,16 +116,15 @@ export async function generateMetadata({ // base-domain URLs — this makes the page agree with it.) const canonical = `${baseUrl}/coupons/${encodeURIComponent(base)}` // Stores with zero visible coupons stay reachable (the prose section - // renders an honest empty state) but are noindexed: thousands of thin - // "no codes right now" pages in the index are soft-404 bloat. cache() - // makes this share one catalog read with the page body. - const { total } = await fetchStoreCoupons(storeParam) + // renders an honest empty state) but are noindexed (verdict.reason === + // 'no-coupons'): thousands of thin "no codes right now" pages in the + // index are soft-404 bloat. return { title, description, alternates: { canonical }, - robots: total === 0 ? { index: false, follow: true } : undefined, + robots, openGraph: { type: 'website', url: canonical, diff --git a/apps/caramel-app/src/app/(marketing)/sources/page.tsx b/apps/caramel-app/src/app/(marketing)/sources/page.tsx index 6d44b9a8..3712deff 100644 --- a/apps/caramel-app/src/app/(marketing)/sources/page.tsx +++ b/apps/caramel-app/src/app/(marketing)/sources/page.tsx @@ -2,6 +2,7 @@ import { listActiveSources } from '@/lib/couponsRepo' import { BASE_URL } from '@/lib/env.client' import { toSourceMetrics } from '@/lib/sourceMetrics' import type { Metadata } from 'next' +import { cache } from 'react' import SourcesPageClient from './SourcesPageClient' // This page reads the coupon catalog from Postgres, and the production image @@ -17,7 +18,7 @@ const base = BASE_URL const canonicalUrl = `${base}/sources` const banner = `${base}/caramel_banner.png` -export const metadata: Metadata = { +const baseMetadata: Metadata = { title, description, alternates: { @@ -48,11 +49,29 @@ export const metadata: Metadata = { }, } +// cache(): generateMetadata needs the active-source list too (for the +// empty-table noindex below), and React request-level caching makes that share +// ONE catalog read with the page body — same pattern as the store page's +// fetchStoreCoupons. +const fetchActiveSources = cache(async () => listActiveSources()) + +export async function generateMetadata(): Promise { + // With zero ACTIVE sources the page is an empty shell (prod had 0 on + // 2026-09-11 and GSC reported it crawled-not-indexed). Keep it reachable + // and followed, but out of the index until there is a table to index; the + // sitemap (app/sitemap.ts) lists /sources under the same condition. + const sources = await fetchActiveSources() + return { + ...baseMetadata, + robots: sources.length > 0 ? undefined : { index: false, follow: true }, + } +} + export default async function SourcesPage() { // SEO: fetch the initial table server-side (same read + mapper as // /api/sources) so crawlers get the populated HTML instead of the old // client-fetch "Loading..." shell. The client keeps refetching through // the API after a source submission. - const initialSources = toSourceMetrics(await listActiveSources()) + const initialSources = toSourceMetrics(await fetchActiveSources()) return } diff --git a/apps/caramel-app/src/app/sitemap.ts b/apps/caramel-app/src/app/sitemap.ts index 44d44a55..e316b2c3 100644 --- a/apps/caramel-app/src/app/sitemap.ts +++ b/apps/caramel-app/src/app/sitemap.ts @@ -1,54 +1,80 @@ -import { listStoreOptions } from '@/lib/couponsRepo' +import { listActiveSources, listStoreSitemapEntries } from '@/lib/couponsRepo' import { BASE_URL } from '@/lib/env.client' +import { collapseStoreRows } from '@/lib/seo/sitemapStores' import type { MetadataRoute } from 'next' // The store half of this sitemap reads the coupon catalog from Postgres, and // the production image builds against a deliberately unreachable placeholder // DATABASE_URL (see the Dockerfile's `.invalid` builder env) — so this route // must be rendered per-request, never prerendered at build time. Crawlers hit -// it rarely and the read is a single indexed DISTINCT, so per-request is cheap. +// it rarely and the read is a single indexed GROUP BY, so per-request is cheap. export const dynamic = 'force-dynamic' const origin = BASE_URL.replace(/\/+$/, '') -// Upper bound on `/coupons/[store]` entries. The sitemap spec caps a single -// file at 50,000 URLs; this stays well under it and bounds the query. If the -// catalog ever outgrows it, the fix is a sitemap index, not a bigger number. +// Upper bound on grouped `coupons.site` rows feeding `/coupons/[store]` +// entries. The sitemap spec caps a single file at 50,000 URLs; this stays well +// under it and bounds the query. If the catalog ever outgrows it, the fix is a +// sitemap index, not a bigger number. const STORE_URL_LIMIT = 5000 -// Public marketing routes. Auth pages ((auth)/login, signup, verify) and -// /profile are deliberately absent — they are disallowed in robots.ts. -const STATIC_ROUTES: ReadonlyArray<{ +type StaticRoute = { path: string changeFrequency: MetadataRoute.Sitemap[number]['changeFrequency'] priority: number -}> = [ +} + +// Public marketing routes. Auth pages ((auth)/login, signup, verify) and +// /profile are deliberately absent — they are disallowed in robots.ts. +// /support is indexable and header-linked, so it belongs here (it was missing +// until 2026-09; GSC saw it only through links). +const STATIC_ROUTES: ReadonlyArray = [ { path: '/', changeFrequency: 'weekly', priority: 1 }, { path: '/coupons', changeFrequency: 'daily', priority: 0.9 }, { path: '/supported-stores', changeFrequency: 'weekly', priority: 0.8 }, { path: '/pricing', changeFrequency: 'monthly', priority: 0.7 }, - { path: '/sources', changeFrequency: 'weekly', priority: 0.6 }, + { path: '/support', changeFrequency: 'monthly', priority: 0.5 }, { path: '/privacy', changeFrequency: 'yearly', priority: 0.3 }, ] +// /sources renders a table of ACTIVE sources. With none (prod had 0 on +// 2026-09-11 — `/api/sources` returned `[]`) it is an empty shell that the page +// itself noindexes ((marketing)/sources/page.tsx), so it is listed only when +// there is something to index. Same read the page uses. +const SOURCES_ROUTE: StaticRoute = { + path: '/sources', + changeFrequency: 'weekly', + priority: 0.6, +} + export default async function sitemap(): Promise { - // Same read the /api/coupons/stores autocomplete uses: DISTINCT visible - // sites, empty query = no ILIKE filter. No `lastModified` is emitted for - // store pages because this row shape carries no timestamp — an invented - // date is worse than none. - const storeRows = await listStoreOptions('', STORE_URL_LIMIT) - const stores = storeRows - .map(row => row.site) - .filter((site): site is string => Boolean(site && site.trim())) + // One aggregate row per raw `coupons.site` (visible coupons only), then + // collapsed to the CANONICAL registrable domain the page canonicalizes to + // and filtered by the SAME indexability policy the page's robots meta + // uses (src/lib/seo/storeIndexability.ts) — so every store here is + // its own canonical and never a noindexed page. `lastModified` is the + // newest `coupons.updated_at` folded into that base: a real catalog + // timestamp, the freshness signal Google needs to re-read a sitemap. + const [storeRows, activeSources] = await Promise.all([ + listStoreSitemapEntries(STORE_URL_LIMIT), + listActiveSources(), + ]) + const stores = collapseStoreRows(storeRows) + + const staticRoutes: StaticRoute[] = + activeSources.length > 0 + ? [...STATIC_ROUTES, SOURCES_ROUTE] + : [...STATIC_ROUTES] return [ - ...STATIC_ROUTES.map(route => ({ + ...staticRoutes.map(route => ({ url: `${origin}${route.path}`, changeFrequency: route.changeFrequency, priority: route.priority, })), - ...stores.map(site => ({ - url: `${origin}/coupons/${encodeURIComponent(site)}`, + ...stores.map(store => ({ + url: `${origin}/coupons/${encodeURIComponent(store.base)}`, + ...(store.lastModified ? { lastModified: store.lastModified } : {}), changeFrequency: 'daily' as const, priority: 0.7, })), diff --git a/apps/caramel-app/src/lib/catalog/applyCatalogRows.ts b/apps/caramel-app/src/lib/catalog/applyCatalogRows.ts index 1e7ddfd9..f4185871 100644 --- a/apps/caramel-app/src/lib/catalog/applyCatalogRows.ts +++ b/apps/caramel-app/src/lib/catalog/applyCatalogRows.ts @@ -102,6 +102,21 @@ class TombstoneGateError extends Error { } } +/** + * The ONE write-side normalization in this engine: `site` is stored lowercase. + * Domains are case-insensitive, but every store read matches + * `site = $base OR site LIKE '%.' || $base` case-SENSITIVELY (plain equality + * keeps `coupons_site_idx` usable) against a base resolveStoreDomain has + * lowercased — so a mixed-case producer value was unreachable from its own + * canonical page (prod 2026-09-11: `eNasco.com`, `Brooklinen.com`). The + * `lowercase_coupon_sites` migration backfilled existing rows; this keeps new + * ones honest. Everything else (status, discount_type, code) stays RAW per + * ingestSchemas.ts — this is a domain-name identity, not a vocabulary. + */ +function normalizeSite(site: string | null): string | null { + return site == null ? null : site.toLowerCase() +} + function chunk(items: readonly T[], size: number): T[][] { const out: T[][] = [] for (let i = 0; i < items.length; i += size) { @@ -339,7 +354,7 @@ export async function applyCatalogRows( for (const rowChunk of chunk(toWrite, WRITE_CHUNK)) { const tuples = rowChunk.map( r => - Prisma.sql`(${r.id}, ${r.code}, ${r.site}, ${r.title}, ${r.description}, ${r.rating}, ${r.discount_type}, ${r.discount_amount}, ${r.expiry}, ${r.expired}, ${r.times_used}, ${r.last_time_used ?? null}, ${r.status}, ${r.verification_message}, ${r.created_at ?? new Date()}, ${r.updated_at})`, + Prisma.sql`(${r.id}, ${r.code}, ${normalizeSite(r.site)}, ${r.title}, ${r.description}, ${r.rating}, ${r.discount_type}, ${r.discount_amount}, ${r.expiry}, ${r.expired}, ${r.times_used}, ${r.last_time_used ?? null}, ${r.status}, ${r.verification_message}, ${r.created_at ?? new Date()}, ${r.updated_at})`, ) await tx.$executeRaw( Prisma.sql` diff --git a/apps/caramel-app/src/lib/couponsDb.ts b/apps/caramel-app/src/lib/couponsDb.ts index fd897602..d0ac9b48 100644 --- a/apps/caramel-app/src/lib/couponsDb.ts +++ b/apps/caramel-app/src/lib/couponsDb.ts @@ -135,6 +135,21 @@ export const SiteCountRowSchema = z.object({ }) export type SiteCountRow = z.infer +/** + * app/sitemap.ts — one row per RAW `coupons.site` with its visible-coupon + * count and newest `updated_at`. `site` is non-null here because the query + * filters `site IS NOT NULL` (a per-query guarantee, like CouponListRow's); + * `last_updated` is `MAX(updated_at)` over a NOT NULL `timestamp(3)` column, so + * a group always has one — `z.coerce.date()` for the same driver-tolerance + * reason RecentStoreRowSchema.added_at uses it. + */ +export const SiteAggregateRowSchema = z.object({ + site: z.string(), + coupon_count: z.number(), + last_updated: z.coerce.date(), +}) +export type SiteAggregateRow = z.infer + /** * supported-stores/page.tsx's "Recently added" strip — one row per store that * has just become supported, newest first. diff --git a/apps/caramel-app/src/lib/couponsRepo.ts b/apps/caramel-app/src/lib/couponsRepo.ts index c4f57e8a..9be417e7 100644 --- a/apps/caramel-app/src/lib/couponsRepo.ts +++ b/apps/caramel-app/src/lib/couponsRepo.ts @@ -40,6 +40,8 @@ import { DiscountTypeRowSchema, type RecentStoreRow, RecentStoreRowSchema, + type SiteAggregateRow, + SiteAggregateRowSchema, type SiteCountRow, SiteCountRowSchema, type SiteRow, @@ -125,8 +127,14 @@ export async function listCoupons( const conditions: Prisma.Sql[] = [visibleCouponsWhere()] if (baseSite) { + // `site` is stored lowercase (applyCatalogRows + the + // lowercase_coupon_sites migration) and this predicate is case- + // sensitive on purpose — plain equality keeps using coupons_site_idx. + // Lowercasing the bound value is the one-line guard that keeps a + // mixed-case caller from matching nothing. + const base = baseSite.toLowerCase() conditions.push( - Prisma.sql`(site = ${baseSite} OR site LIKE ${'%.' + baseSite})`, + Prisma.sql`(site = ${base} OR site LIKE ${'%.' + base})`, ) } @@ -202,6 +210,11 @@ export async function listStoreCoupons( // client fetch agree (no hydration flash) — see lib/coupons.ts's // VISIBLE_COUPON_STATUSES doc comment for the full rationale. const visible = visibleCouponsWhere() + // Same one-line guard as listCoupons: the column is lowercase, the + // predicate is case-sensitive (index-friendly), so the bound base must be + // lowercase too. resolveStoreDomain already lowercases for the page; this + // makes the read correct for any caller. + const base = baseSite.toLowerCase() const [rawCoupons, rawTotalRow] = await Promise.all([ prisma.$queryRaw(Prisma.sql` SELECT id, code, site, title, description, rating, @@ -210,14 +223,14 @@ export async function listStoreCoupons( status, verification_message AS "verificationMessage" FROM coupons WHERE ${visible} - AND (site = ${baseSite} OR site LIKE ${'%.' + baseSite}) + AND (site = ${base} OR site LIKE ${'%.' + base}) ORDER BY ${rankingOrderSql()} LIMIT ${limit} `), prisma.$queryRaw(Prisma.sql` SELECT COUNT(*)::int AS total FROM coupons WHERE ${visible} - AND (site = ${baseSite} OR site LIKE ${'%.' + baseSite}) + AND (site = ${base} OR site LIKE ${'%.' + base}) `), ]) const coupons = parseCouponRows( @@ -310,6 +323,39 @@ export async function listStoreOptions( return parseCouponRows(SiteRowSchema, rawRows, 'coupons.stores') } +/** + * app/sitemap.ts — one aggregate row per RAW `coupons.site` among VISIBLE + * coupons: its visible-coupon count and its newest `updated_at`. + * + * Deliberately NOT collapsed to the registrable domain here: the slug→base + * rule lives in exactly one place (src/lib/storeDomain.ts's resolveStoreDomain, + * Public-Suffix-List backed) and src/lib/seo/sitemapStores.ts applies it plus + * the shared indexability policy to these rows. That keeps this read a plain + * indexed GROUP BY (no SQL re-implementation of the PSL) and keeps the sitemap's + * "which base, is it indexable" decisions testable without a database. + * + * `MAX(updated_at)` is the sitemap's `` source — a real catalog + * timestamp (the producer's last-write stamp), never an invented date. + * `visibleCouponsWhere()` is the same predicate the store page counts with, so + * a site that appears here has ≥1 visible coupon under that raw slug. + * `site IS NOT NULL` makes the row's `site` non-null (SiteAggregateRowSchema). + */ +export async function listStoreSitemapEntries( + limit: number, +): Promise { + const rawRows = await prisma.$queryRaw(Prisma.sql` + SELECT site, + COUNT(*)::int AS coupon_count, + MAX(updated_at) AS last_updated + FROM coupons + WHERE ${visibleCouponsWhere()} AND site IS NOT NULL + GROUP BY site + ORDER BY site ASC + LIMIT ${limit} + `) + return parseCouponRows(SiteAggregateRowSchema, rawRows, 'sitemap.stores') +} + /** api/coupons/filters/route.ts GET — sites half. The route's `includeSites` gate stays there (calls this only when true); this fn only owns its own `sitesLimit<=0` short-circuit. */ export async function listFilterSites(sitesLimit: number): Promise { if (sitesLimit <= 0) return [] diff --git a/apps/caramel-app/src/lib/seo/sitemapStores.ts b/apps/caramel-app/src/lib/seo/sitemapStores.ts new file mode 100644 index 00000000..b750d018 --- /dev/null +++ b/apps/caramel-app/src/lib/seo/sitemapStores.ts @@ -0,0 +1,82 @@ +// src/lib/seo/sitemapStores.ts +// +// Collapses the catalog's per-`site` aggregate rows into the CANONICAL store +// entries the sitemap emits — one entry per registrable domain, the same key +// the page canonicalizes to (resolveStoreDomain), gated by the same policy the +// page's robots meta uses (evaluateStorePageIndexability). Pure: no DB, no env. +// +// Measured on the served sitemap 2026-09-11: 4,311 raw `coupons.site` slugs +// collapse to 4,262 canonical bases — 84 subdomain slugs (athleta.gap.com, +// au.shein.com, shop.nhl.com …) fold into their base, 2 mixed-case slugs +// (Brooklinen.com, eNasco.com) fold into their lowercase twin, 40 bases were +// listed under several slugs at once (gap.com ×4, shein.com ×4, att.com ×3), +// 38 canonical targets (mattel.com, nfl.com, enasco.com, w3schools.com …) were +// missing entirely, and 1 slug (dhl.com-us-en-home.html) is not a domain. +import { evaluateStorePageIndexability } from '@/lib/seo/storeIndexability' +import { resolveStoreDomain } from '@/lib/storeDomain' + +/** One `GROUP BY site` row from couponsRepo.listStoreSitemapEntries. */ +export type StoreSitemapRow = { + site: string | null + coupon_count: number + last_updated: Date | null +} + +/** One canonical `/coupons/` sitemap entry. */ +export type StoreSitemapEntry = { + base: string + couponCount: number + /** Newest `coupons.updated_at` across every slug folded into this base. */ + lastModified: Date | null +} + +function newerOf(a: Date | null, b: Date | null): Date | null { + if (!a) return b + if (!b) return a + return b.getTime() > a.getTime() ? b : a +} + +/** + * Group rows by their registrable domain, summing counts and keeping the + * newest timestamp, then keep only the bases the indexability policy admits. + * Rows whose `site` resolves to no store (null, '', a bare public suffix, a + * non-domain string) are dropped — the page would `noindex` them anyway. + * Output is sorted by base for a stable, diffable sitemap. + */ +export function collapseStoreRows( + rows: ReadonlyArray, +): StoreSitemapEntry[] { + const byBase = new Map() + for (const row of rows) { + if (!row.site) continue + const base = resolveStoreDomain(row.site) + if (!base) continue + const count = Number.isFinite(row.coupon_count) + ? Math.max(0, row.coupon_count) + : 0 + const existing = byBase.get(base) + if (existing) { + existing.couponCount += count + existing.lastModified = newerOf( + existing.lastModified, + row.last_updated, + ) + } else { + byBase.set(base, { + base, + couponCount: count, + lastModified: row.last_updated, + }) + } + } + + return Array.from(byBase.values()) + .filter( + entry => + evaluateStorePageIndexability({ + base: entry.base, + visibleCouponCount: entry.couponCount, + }).indexable, + ) + .sort((a, b) => (a.base < b.base ? -1 : a.base > b.base ? 1 : 0)) +} diff --git a/apps/caramel-app/src/lib/seo/storeIndexability.ts b/apps/caramel-app/src/lib/seo/storeIndexability.ts new file mode 100644 index 00000000..bc131d9b --- /dev/null +++ b/apps/caramel-app/src/lib/seo/storeIndexability.ts @@ -0,0 +1,66 @@ +// src/lib/seo/storeIndexability.ts +// +// THE ONE indexability policy for `/coupons/[store]` pages. Both consumers — +// the sitemap (src/app/sitemap.ts, via sitemapStores.ts) and the page's own +// generateMetadata ((marketing)/coupons/[store]/page.tsx) — consult THIS +// function, so "is this store page in the sitemap" and "does this store page +// carry noindex" can never disagree again. +// +// Why it exists (GSC, 2026-09-11): the sitemap listed 4,317 store URLs and +// Google knew 4,289 of them as "unknown". Three independent computations had +// drifted apart: the sitemap emitted raw `coupons.site` slugs (84 subdomains +// like athleta.gap.com, 2 mixed-case like eNasco.com, 1 non-domain), the page +// canonicalized every slug to its registrable base, and the page's noindex +// rule ran on a count the sitemap never looked at — so the sitemap happily +// listed pages whose canonical pointed elsewhere and pages that served +// `noindex`. See caramel-artifact/seo-2026-09-11/audit-findings.md §Sitemap. +// +// Pure by design: no DB, no env, no I/O. The caller resolves the slug through +// resolveStoreDomain() and counts visible coupons; this function only decides. + +/** Why a store page is NOT indexable. `null` means it is. */ +export type StoreIndexabilityReason = 'not-a-store' | 'no-coupons' | null + +export type StoreIndexabilityInput = { + /** + * The registrable domain the page canonicalizes to — resolveStoreDomain() + * output. `null`/'' means the slug names no real store (a bare public + * suffix like `co.uk`, a garbage path like `dhl.com-us-en-home.html`, an + * unregistrable TLD) and the page can only render its empty state. + */ + base: string | null | undefined + /** + * Visible coupons (visibleCouponsWhere) for that base — the SAME number + * the page's prose renders. Zero-coupon pages are soft-404 bloat. + */ + visibleCouponCount: number +} + +export type StoreIndexability = { + indexable: boolean + reason: StoreIndexabilityReason +} + +/** + * A store page is indexable exactly when it names a real store AND currently + * lists at least one visible coupon. Everything else is `noindex, follow` on + * the page and absent from the sitemap. + * + * ONE coupon is enough: a page with a single live code is still a real answer + * to " coupon code", and the catalog's counts move daily — raising the + * bar would churn thousands of URLs in and out of the sitemap as codes expire + * and reappear, which is worse for crawl budget than a few thin-but-true pages. + */ +export function evaluateStorePageIndexability( + input: StoreIndexabilityInput, +): StoreIndexability { + const base = typeof input.base === 'string' ? input.base.trim() : '' + if (!base) return { indexable: false, reason: 'not-a-store' } + if ( + !Number.isFinite(input.visibleCouponCount) || + input.visibleCouponCount < 1 + ) { + return { indexable: false, reason: 'no-coupons' } + } + return { indexable: true, reason: null } +} diff --git a/apps/caramel-app/tests/integration/coupons-read.itest.ts b/apps/caramel-app/tests/integration/coupons-read.itest.ts index 890ead02..b70b3ac6 100644 --- a/apps/caramel-app/tests/integration/coupons-read.itest.ts +++ b/apps/caramel-app/tests/integration/coupons-read.itest.ts @@ -3,6 +3,7 @@ import { getCouponStats, listActiveSources, listCoupons, + listStoreSitemapEntries, listSupportedStoreConfigs, } from '@/lib/couponsRepo' import prisma from '@/lib/prisma' @@ -199,3 +200,42 @@ describe('listActiveSources — sources ⋈ coupons aggregates (real pg :58005)' expect(feedA.total_expired).toBeGreaterThanOrEqual(2) }) }) + +describe('listStoreSitemapEntries — per-site visible aggregates for the sitemap (real pg :58005)', () => { + it('returns one row per visible site, sorted by site, with an ::int count that matches listCoupons and a real Date last_updated', async () => { + const rows = await listStoreSitemapEntries(5000) + + // Non-null sites only, ascending — the GROUP BY / ORDER BY held for real. + const sites = rows.map(r => r.site) + for (const site of sites) expect(site).toBeTruthy() + expect(sites).toEqual([...sites].sort()) + + // codecademy.com is the store no other suite writes to, so its + // aggregate is asserted exactly against the SAME visibility predicate + // listCoupons applies: the sitemap count must equal the page count. + const codecademy = rows.find(r => r.site === 'codecademy.com') + expect(codecademy).toBeDefined() + expect(Number.isInteger(codecademy!.coupon_count)).toBe(true) + expect(codecademy!.coupon_count).toBeGreaterThan(0) + expect(codecademy!.last_updated).toBeInstanceOf(Date) + expect(Number.isNaN(codecademy!.last_updated.getTime())).toBe(false) + const { total } = await listCoupons({ + baseSite: 'codecademy.com', + limit: 1, + skip: 0, + }) + // listCoupons also matches subdomain rows (LIKE '%.codecademy.com'), + // so compare against the sum over every raw site under that base. + const underBase = rows + .filter( + r => + r.site === 'codecademy.com' || + r.site.endsWith('.codecademy.com'), + ) + .reduce((sum, r) => sum + r.coupon_count, 0) + expect(underBase).toBe(total) + + // LIMIT is bound (the sitemap's 5000 cap is a real bound, not decoration). + expect(await listStoreSitemapEntries(2)).toHaveLength(2) + }) +}) diff --git a/apps/caramel-app/tests/integration/ingest-catalog.itest.ts b/apps/caramel-app/tests/integration/ingest-catalog.itest.ts index 3b96f594..35f7beef 100644 --- a/apps/caramel-app/tests/integration/ingest-catalog.itest.ts +++ b/apps/caramel-app/tests/integration/ingest-catalog.itest.ts @@ -1,6 +1,7 @@ import { applyCatalogRows } from '@/lib/catalog/applyCatalogRows' import type { IngestCatalogPayload } from '@/lib/catalog/ingestSchemas' import { VISIBLE_COUPON_STATUSES } from '@/lib/coupons' +import { listStoreCoupons, listStoreSitemapEntries } from '@/lib/couponsRepo' import prisma from '@/lib/prisma' import { afterAll, afterEach, describe, expect, it } from 'vitest' @@ -251,3 +252,46 @@ describe('applyCatalogRows — transaction atomicity', () => { ).toBeNull() }) }) + +describe('applyCatalogRows — site is stored lowercase, so the canonical (lowercase) store page finds it', () => { + // Prod 2026-09-11: `eNasco.com` rows were unreachable from /coupons/enasco.com + // AND /coupons/eNasco.com (case-sensitive `site = $base` on a lowercased + // base). A private `.example`-free but still made-up host so it cannot + // collide with the seed or any other suite's rows. + const mixedCase = 'ITest-MixedCase-Store.test' + const lower = 'itest-mixedcase-store.test' + + it('stores the lowercase site, and listStoreCoupons/listStoreSitemapEntries see it under the lowercase base', async () => { + const id = '800000090' + const r = await applyCatalogRows( + push([coupon(id, '2026-07-14T12:00:00.000Z', { site: mixedCase })]), + ) + expect(r.gated).toBe(false) + + expect((await prisma.coupon.findUnique({ where: { id } }))?.site).toBe( + lower, + ) + + // The page read: a mixed-case caller is lowercased before binding. + const viaMixed = await listStoreCoupons(mixedCase, 5) + const viaLower = await listStoreCoupons(lower, 5) + expect(viaLower.total).toBe(1) + expect(viaMixed.total).toBe(1) + expect(viaLower.coupons[0]?.id).toBe(id) + + // The sitemap read groups it under the lowercase site. + const entries = await listStoreSitemapEntries(5000) + expect(entries.find(e => e.site === lower)?.coupon_count).toBe(1) + expect(entries.find(e => e.site === mixedCase)).toBeUndefined() + }) + + it('a null site stays null (the column is nullable; nothing is invented)', async () => { + const id = '800000091' + await applyCatalogRows( + push([coupon(id, '2026-07-14T12:00:00.000Z', { site: null })]), + ) + expect( + (await prisma.coupon.findUnique({ where: { id } }))?.site, + ).toBeNull() + }) +}) diff --git a/apps/caramel-app/tests/unit/applyCatalogRows-site-case.test.ts b/apps/caramel-app/tests/unit/applyCatalogRows-site-case.test.ts new file mode 100644 index 00000000..daa8645d --- /dev/null +++ b/apps/caramel-app/tests/unit/applyCatalogRows-site-case.test.ts @@ -0,0 +1,95 @@ +import { applyCatalogRows } from '@/lib/catalog/applyCatalogRows' +import type { IngestCatalogPayload } from '@/lib/catalog/ingestSchemas' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +// Pins the ONE write-side normalization applyCatalogRows performs: `site` is +// stored lowercased. Domains are case-insensitive, but the store-page reads +// match `site = $base OR site LIKE '%.' || $base` case-SENSITIVELY on a +// lowercased $base (resolveStoreDomain), so a mixed-case producer value +// (prod 2026-09-11: `eNasco.com`, `Brooklinen.com`) was unreachable from its +// own canonical page. The transaction is mocked so no DB is touched; the +// INSERT's bound VALUES are captured off the composed Prisma.Sql and asserted +// directly. The real round-trip is pinned against live Postgres in +// tests/integration/ingest-catalog.itest.ts. + +type Captured = { sql: string; values: unknown[] } + +const { txMock, captured } = vi.hoisted(() => { + const captured: Captured[] = [] + const txMock = { + // No pre-existing rows for any pushed id → every row is an insert. + $queryRaw: vi.fn(async () => []), + coupon: { count: vi.fn(async () => 0) }, + $executeRaw: vi.fn(async (arg: Captured) => { + captured.push({ sql: arg.sql, values: arg.values }) + return 1 + }), + } + return { txMock, captured } +}) + +vi.mock('@/lib/prisma', () => ({ + default: { + $transaction: (fn: (tx: typeof txMock) => Promise) => + fn(txMock), + }, +})) + +type IngestCoupon = IngestCatalogPayload['coupons'][number] + +function coupon(id: string, site: string | null): IngestCoupon { + return { + id, + code: `CODE-${id}`, + site, + title: `t-${id}`, + description: 'd', + rating: 0, + discount_type: null, + discount_amount: null, + expiry: null, + expired: false, + times_used: 0, + last_time_used: null, + status: 'valid', + verification_message: null, + updated_at: new Date('2026-09-12T00:00:00.000Z'), + } +} + +beforeEach(() => { + captured.length = 0 + txMock.$queryRaw.mockClear() + txMock.$executeRaw.mockClear() +}) + +describe('applyCatalogRows — site is lowercased on write', () => { + it('binds the lowercase site for mixed-case producer values and leaves a null site null', async () => { + const result = await applyCatalogRows({ + coupons: [ + coupon('800000101', 'eNasco.com'), + coupon('800000102', 'Brooklinen.com'), + coupon('800000103', 'already-lower.com'), + coupon('800000104', null), + ], + storeConfigs: [], + sources: [], + force: false, + }) + expect(result.gated).toBe(false) + + const insert = captured.find(c => c.sql.includes('INSERT INTO coupons')) + expect(insert).toBeDefined() + const bound = insert!.values + + expect(bound).toContain('enasco.com') + expect(bound).toContain('brooklinen.com') + expect(bound).toContain('already-lower.com') + expect(bound).not.toContain('eNasco.com') + expect(bound).not.toContain('Brooklinen.com') + // The nullable column stays nullable — null is not turned into ''. + expect(bound).toContain(null) + // Only `site` is normalized: the code keeps its producer casing. + expect(bound).toContain('CODE-800000101') + }) +}) diff --git a/apps/caramel-app/tests/unit/couponsRepo.test.ts b/apps/caramel-app/tests/unit/couponsRepo.test.ts index 366533c5..5caaf718 100644 --- a/apps/caramel-app/tests/unit/couponsRepo.test.ts +++ b/apps/caramel-app/tests/unit/couponsRepo.test.ts @@ -2,6 +2,8 @@ import { expireCoupons, getCouponStats, listCoupons, + listStoreCoupons, + listStoreSitemapEntries, requestSource, } from '@/lib/couponsRepo' import { beforeEach, describe, expect, it, vi } from 'vitest' @@ -23,6 +25,9 @@ let rules: MockRule[] = [] // Raw SQL text of every DB call, in order — lets a test assert on the // generated QUERY SHAPE itself. let capturedQueries: string[] = [] +// The bound parameter VALUES of every $queryRaw, in the same order — lets a +// test assert on what was actually bound (e.g. the lowercased store base). +let capturedValues: unknown[][] = [] // Affected-row count the mocked $executeRaw returns (expire). let executeRawResult = 0 // Args every prisma.source.create() was called with (requestSource). @@ -35,8 +40,9 @@ function mockRows(match: (sql: string) => boolean, rows: unknown[]) { // composed, flattened query text (nested fragments inlined, values as `?`). vi.mock('@/lib/prisma', () => ({ default: { - $queryRaw: (arg: { sql: string }) => { + $queryRaw: (arg: { sql: string; values: unknown[] }) => { capturedQueries.push(arg.sql) + capturedValues.push(arg.values) const rows = rules.find(r => r.match(arg.sql))?.rows ?? [] return Promise.resolve(rows) }, @@ -57,6 +63,7 @@ vi.mock('@/lib/prisma', () => ({ beforeEach(() => { rules = [] capturedQueries = [] + capturedValues = [] executeRawResult = 0 capturedSourceCreates = [] }) @@ -217,3 +224,107 @@ describe('requestSource (write, app sources INSERT via prisma.source.create)', ( expect(arg.data.id.length).toBeGreaterThan(0) }) }) + +describe('listStoreSitemapEntries (app/sitemap.ts read)', () => { + it('groups VISIBLE sited coupons per raw site with an ::int count and MAX(updated_at), ordered by site, LIMIT-bound', async () => { + mockRows( + sql => sql.includes('MAX(updated_at)'), + [ + { + site: 'athleta.gap.com', + coupon_count: 7, + last_updated: new Date('2026-09-01T00:00:00.000Z'), + }, + { + site: 'gap.com', + coupon_count: 30, + // A driver handing the timestamp back as a string is + // normalized, not treated as drift (z.coerce.date). + last_updated: '2026-08-15T00:00:00.000Z', + }, + ], + ) + + const rows = await listStoreSitemapEntries(5000) + + expect(rows).toEqual([ + { + site: 'athleta.gap.com', + coupon_count: 7, + last_updated: new Date('2026-09-01T00:00:00.000Z'), + }, + { + site: 'gap.com', + coupon_count: 30, + last_updated: new Date('2026-08-15T00:00:00.000Z'), + }, + ]) + + // Query shape: the shared visibility predicate (the same fragment + // every listing read inlines), sited rows only, grouped per raw site. + expect(capturedQueries).toHaveLength(1) + const q = capturedQueries[0]! + expect(q).toContain('COUNT(*)::int AS coupon_count') + expect(q).toContain('MAX(updated_at) AS last_updated') + expect(q).toContain('expired = FALSE') + expect(q).toContain('status IN (') + expect(q).toContain('site IS NOT NULL') + expect(q).toContain('GROUP BY site') + expect(q).toContain('ORDER BY site ASC') + expect(q).toContain('LIMIT ?') + expect(capturedValues[0]).toContain(5000) + }) + + it('a row missing coupon_count fails the zod boundary loudly (drift, not silent zeros)', async () => { + mockRows( + sql => sql.includes('MAX(updated_at)'), + [{ site: 'gap.com', last_updated: new Date() }], + ) + await expect(listStoreSitemapEntries(10)).rejects.toThrow( + /coupons-db schema drift \[sitemap\.stores\]/, + ) + }) +}) + +describe('store-matching reads bind the LOWERCASE base (site column is stored lowercase)', () => { + it('listStoreCoupons("eNasco.com") binds enasco.com / %.enasco.com in BOTH the list and the count query', async () => { + mockRows( + sql => sql.includes('FROM coupons') && sql.includes('LIMIT'), + [], + ) + mockRows(sql => sql.includes('COUNT(*)::int AS total'), [{ total: 0 }]) + + await listStoreCoupons('eNasco.com', 5) + + expect(capturedValues).toHaveLength(2) + for (const values of capturedValues) { + expect(values).toContain('enasco.com') + expect(values).toContain('%.enasco.com') + expect(values).not.toContain('eNasco.com') + expect(values).not.toContain('%.eNasco.com') + } + // The predicate itself stays plain, index-friendly equality — no + // LOWER(site) wrapper that would defeat coupons_site_idx. + for (const q of capturedQueries) { + expect(q).toMatch(/\(site = \? OR site LIKE \?\)/) + expect(q).not.toMatch(/lower\(site\)/i) + } + }) + + it('listCoupons({ baseSite: "Brooklinen.com" }) binds brooklinen.com', async () => { + mockRows( + sql => sql.includes('FROM coupons') && sql.includes('LIMIT'), + [], + ) + mockRows(sql => sql.includes('COUNT(*)'), [{ total: 0 }]) + + await listCoupons({ baseSite: 'Brooklinen.com', limit: 10, skip: 0 }) + + expect(capturedValues).toHaveLength(2) + for (const values of capturedValues) { + expect(values).toContain('brooklinen.com') + expect(values).toContain('%.brooklinen.com') + expect(values).not.toContain('Brooklinen.com') + } + }) +}) diff --git a/apps/caramel-app/tests/unit/sitemap.test.ts b/apps/caramel-app/tests/unit/sitemap.test.ts new file mode 100644 index 00000000..22a84281 --- /dev/null +++ b/apps/caramel-app/tests/unit/sitemap.test.ts @@ -0,0 +1,221 @@ +import sitemap from '@/app/sitemap' +import type { MetadataRoute } from 'next' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +// Pins src/app/sitemap.ts's composition contract (there was NO unit test for +// it until 2026-09 — the served sitemap drifted to 84 subdomain slugs, 2 +// mixed-case slugs, 1 non-domain and 38 missing canonical targets before GSC +// told us; see caramel-artifact/seo-2026-09-11/audit-findings.md §Sitemap). +// +// BASE_URL is resolved at module scope in sitemap.ts, so it is mocked +// STATICALLY (one origin for the whole file) rather than per case like +// robots-env-contract.test.ts does: the store collapse loads tldts's Public +// Suffix List, which is far too heavy to re-import under resetModules for +// every case. The two catalog reads are mocked at the couponsRepo boundary so +// no SQL runs. + +const ORIGIN = 'https://grabcaramel.com' + +const { repoMock } = vi.hoisted(() => ({ + repoMock: { + listStoreSitemapEntries: vi.fn(), + listActiveSources: vi.fn(), + }, +})) +vi.mock('@/lib/couponsRepo', () => repoMock) +vi.mock('@/lib/env.client', () => ({ BASE_URL: 'https://grabcaramel.com' })) + +async function renderSitemap(): Promise { + return sitemap() +} + +const d = (iso: string) => new Date(iso) + +const ACTIVE_SOURCE = { + id: 'src-1', + source: 'Feed A', + websites: ['gap.com'], + status: 'ACTIVE', + total_coupons: 3, + total_used: 0, + total_expired: 0, +} + +beforeEach(() => { + repoMock.listStoreSitemapEntries.mockReset() + repoMock.listActiveSources.mockReset() + repoMock.listStoreSitemapEntries.mockResolvedValue([]) + repoMock.listActiveSources.mockResolvedValue([]) +}) + +function urlsOf(entries: MetadataRoute.Sitemap): string[] { + return entries.map(e => e.url) +} + +describe('sitemap.ts — static routes', () => { + it('lists the public marketing routes including /support (monthly, 0.5) and never the auth/profile pages', async () => { + const entries = await renderSitemap() + const urls = urlsOf(entries) + + for (const path of [ + '/', + '/coupons', + '/supported-stores', + '/pricing', + '/support', + '/privacy', + ]) { + expect(urls, `missing ${path}`).toContain(`${ORIGIN}${path}`) + } + const support = entries.find(e => e.url === `${ORIGIN}/support`) + expect(support?.changeFrequency).toBe('monthly') + expect(support?.priority).toBe(0.5) + + for (const path of ['/login', '/signup', '/verify', '/profile']) { + expect(urls).not.toContain(`${ORIGIN}${path}`) + } + }) + + it('omits /sources when there are no ACTIVE sources (an empty shell the page itself noindexes)', async () => { + repoMock.listActiveSources.mockResolvedValue([]) + const urls = urlsOf(await renderSitemap()) + expect(urls).not.toContain(`${ORIGIN}/sources`) + }) + + it('emits /sources (weekly, 0.6) when at least one ACTIVE source exists', async () => { + repoMock.listActiveSources.mockResolvedValue([ACTIVE_SOURCE]) + const entries = await renderSitemap() + const sources = entries.find(e => e.url === `${ORIGIN}/sources`) + expect(sources).toBeDefined() + expect(sources?.changeFrequency).toBe('weekly') + expect(sources?.priority).toBe(0.6) + }) +}) + +describe('sitemap.ts — store entries are canonical, lowercase, deduped, policy-gated', () => { + it('emits ONE lowercase base-domain URL per registrable domain, folding subdomain and mixed-case slugs, dropping non-stores and 0-count sites', async () => { + repoMock.listStoreSitemapEntries.mockResolvedValue([ + { + site: 'athleta.gap.com', + coupon_count: 7, + last_updated: d('2026-09-01T00:00:00Z'), + }, + { + site: 'gap.com', + coupon_count: 30, + last_updated: d('2026-08-15T00:00:00Z'), + }, + { + site: 'Brooklinen.com', + coupon_count: 2, + last_updated: d('2026-07-01T00:00:00Z'), + }, + { + site: 'brooklinen.com', + coupon_count: 9, + last_updated: d('2026-09-05T00:00:00Z'), + }, + { + site: 'eNasco.com', + coupon_count: 3, + last_updated: d('2026-08-20T00:00:00Z'), + }, + { + site: 'dhl.com-us-en-home.html', + coupon_count: 5, + last_updated: d('2026-09-01T00:00:00Z'), + }, + { + site: 'co.uk', + coupon_count: 50, + last_updated: d('2026-09-01T00:00:00Z'), + }, + { + site: 'zero-codes.com', + coupon_count: 0, + last_updated: d('2026-09-01T00:00:00Z'), + }, + ]) + + const entries = await renderSitemap() + const storeUrls = urlsOf(entries).filter(u => + u.startsWith(`${ORIGIN}/coupons/`), + ) + + expect(storeUrls).toEqual([ + `${ORIGIN}/coupons/brooklinen.com`, + `${ORIGIN}/coupons/enasco.com`, + `${ORIGIN}/coupons/gap.com`, + ]) + // No duplicates, every slug lowercase. + expect(new Set(storeUrls).size).toBe(storeUrls.length) + for (const url of storeUrls) expect(url).toBe(url.toLowerCase()) + // Never the raw variants. + for (const raw of [ + 'athleta.gap.com', + 'Brooklinen.com', + 'eNasco.com', + 'dhl.com-us-en-home.html', + 'co.uk', + 'zero-codes.com', + ]) { + expect(urlsOf(entries)).not.toContain(`${ORIGIN}/coupons/${raw}`) + } + + // Store entries keep the existing crawl hints. + const gap = entries.find(e => e.url === `${ORIGIN}/coupons/gap.com`) + expect(gap?.changeFrequency).toBe('daily') + expect(gap?.priority).toBe(0.7) + }) + + it('carries lastModified through as the newest updated_at folded into the base, and omits it when no row had one', async () => { + repoMock.listStoreSitemapEntries.mockResolvedValue([ + { + site: 'athleta.gap.com', + coupon_count: 7, + last_updated: d('2026-09-10T12:00:00Z'), + }, + { + site: 'gap.com', + coupon_count: 30, + last_updated: d('2026-08-15T00:00:00Z'), + }, + { site: 'nodate.com', coupon_count: 1, last_updated: null }, + ]) + + const entries = await renderSitemap() + const gap = entries.find(e => e.url === `${ORIGIN}/coupons/gap.com`) + expect(gap?.lastModified).toEqual(d('2026-09-10T12:00:00Z')) + + const nodate = entries.find( + e => e.url === `${ORIGIN}/coupons/nodate.com`, + ) + expect(nodate).toBeDefined() + expect(nodate?.lastModified).toBeUndefined() + // Static routes never invent a date either. + expect( + entries.find(e => e.url === `${ORIGIN}/`)?.lastModified, + ).toBeUndefined() + }) + + it('reads grouped rows under the 5000 cap and encodes the base into the URL', async () => { + repoMock.listStoreSitemapEntries.mockResolvedValue([ + { + site: 'mymemory.co.uk', + coupon_count: 4, + last_updated: d('2026-09-01T00:00:00Z'), + }, + ]) + const entries = await renderSitemap() + expect(repoMock.listStoreSitemapEntries).toHaveBeenCalledWith(5000) + expect(urlsOf(entries)).toContain(`${ORIGIN}/coupons/mymemory.co.uk`) + }) + + it('with an empty catalog emits only the static routes', async () => { + const entries = await renderSitemap() + expect( + urlsOf(entries).some(u => u.startsWith(`${ORIGIN}/coupons/`)), + ).toBe(false) + expect(entries.length).toBe(6) + }) +}) diff --git a/apps/caramel-app/tests/unit/sitemapStores.test.ts b/apps/caramel-app/tests/unit/sitemapStores.test.ts new file mode 100644 index 00000000..0319a014 --- /dev/null +++ b/apps/caramel-app/tests/unit/sitemapStores.test.ts @@ -0,0 +1,172 @@ +import { collapseStoreRows } from '@/lib/seo/sitemapStores' +import { describe, expect, it } from 'vitest' + +// Pins the sitemap's store collapse: raw `GROUP BY site` rows → one canonical +// entry per registrable domain, counts summed, newest timestamp kept, policy +// applied. The fixtures are the real prod shapes from the 2026-09-11 audit +// (caramel-artifact/seo-2026-09-11/audit-findings.md §Sitemap). + +const d = (iso: string) => new Date(iso) + +describe('collapseStoreRows — subdomain slugs fold into ONE canonical base', () => { + it('athleta.gap.com + gap.com + bananarepublic.gap.com → one gap.com entry with the summed count and the max last_updated', () => { + const entries = collapseStoreRows([ + { + site: 'athleta.gap.com', + coupon_count: 7, + last_updated: d('2026-09-01T00:00:00Z'), + }, + { + site: 'gap.com', + coupon_count: 30, + last_updated: d('2026-08-15T00:00:00Z'), + }, + { + site: 'bananarepublic.gap.com', + coupon_count: 4, + last_updated: d('2026-09-10T12:00:00Z'), + }, + ]) + expect(entries).toEqual([ + { + base: 'gap.com', + couponCount: 41, + lastModified: d('2026-09-10T12:00:00Z'), + }, + ]) + }) + + it('a base that only exists as subdomain rows (roborock.com via us.roborock.com) is emitted under the base', () => { + const entries = collapseStoreRows([ + { + site: 'us.roborock.com', + coupon_count: 16, + last_updated: d('2026-09-02T00:00:00Z'), + }, + ]) + expect(entries.map(e => e.base)).toEqual(['roborock.com']) + }) +}) + +describe('collapseStoreRows — mixed-case slugs fold into their lowercase twin', () => { + it('Brooklinen.com + brooklinen.com → one brooklinen.com entry', () => { + const entries = collapseStoreRows([ + { + site: 'Brooklinen.com', + coupon_count: 2, + last_updated: d('2026-07-01T00:00:00Z'), + }, + { + site: 'brooklinen.com', + coupon_count: 9, + last_updated: d('2026-09-05T00:00:00Z'), + }, + ]) + expect(entries).toEqual([ + { + base: 'brooklinen.com', + couponCount: 11, + lastModified: d('2026-09-05T00:00:00Z'), + }, + ]) + }) + + it('eNasco.com alone → enasco.com (the page canonicalizes to lowercase, so the sitemap must too)', () => { + const entries = collapseStoreRows([ + { + site: 'eNasco.com', + coupon_count: 3, + last_updated: d('2026-08-20T00:00:00Z'), + }, + ]) + expect(entries.map(e => e.base)).toEqual(['enasco.com']) + }) +}) + +describe('collapseStoreRows — rows that name no store are dropped', () => { + it('drops dhl.com-us-en-home.html (not a domain), co.uk (bare public suffix), null and empty sites', () => { + const entries = collapseStoreRows([ + { + site: 'dhl.com-us-en-home.html', + coupon_count: 5, + last_updated: d('2026-09-01T00:00:00Z'), + }, + { + site: 'co.uk', + coupon_count: 50, + last_updated: d('2026-09-01T00:00:00Z'), + }, + { site: null, coupon_count: 5, last_updated: null }, + { site: '', coupon_count: 5, last_updated: null }, + { + site: 'mymemory.co.uk', + coupon_count: 12, + last_updated: d('2026-09-01T00:00:00Z'), + }, + ]) + expect(entries.map(e => e.base)).toEqual(['mymemory.co.uk']) + }) + + it('drops a 0-count row (the page would noindex it)', () => { + const entries = collapseStoreRows([ + { + site: 'example.com', + coupon_count: 0, + last_updated: d('2026-09-01T00:00:00Z'), + }, + { + site: 'other.example.com', + coupon_count: 0, + last_updated: null, + }, + { + site: 'kept.com', + coupon_count: 1, + last_updated: d('2026-09-01T00:00:00Z'), + }, + ]) + expect(entries.map(e => e.base)).toEqual(['kept.com']) + }) + + it('a base whose rows sum to ≥1 stays even if one of its slugs is 0', () => { + const entries = collapseStoreRows([ + { site: 'a.shein.com', coupon_count: 0, last_updated: null }, + { + site: 'shein.com', + coupon_count: 1, + last_updated: d('2026-09-01T00:00:00Z'), + }, + ]) + expect(entries).toEqual([ + { + base: 'shein.com', + couponCount: 1, + lastModified: d('2026-09-01T00:00:00Z'), + }, + ]) + }) +}) + +describe('collapseStoreRows — output shape', () => { + it('is sorted by base and carries a null lastModified when no row had a timestamp', () => { + const entries = collapseStoreRows([ + { site: 'zeta.com', coupon_count: 1, last_updated: null }, + { + site: 'alpha.com', + coupon_count: 2, + last_updated: d('2026-09-01T00:00:00Z'), + }, + { site: 'mid.com', coupon_count: 3, last_updated: null }, + ]) + expect(entries.map(e => e.base)).toEqual([ + 'alpha.com', + 'mid.com', + 'zeta.com', + ]) + expect(entries[2]!.lastModified).toBeNull() + }) + + it('returns [] for no rows', () => { + expect(collapseStoreRows([])).toEqual([]) + }) +}) diff --git a/apps/caramel-app/tests/unit/sources-page.test.ts b/apps/caramel-app/tests/unit/sources-page.test.ts new file mode 100644 index 00000000..f8b0f886 --- /dev/null +++ b/apps/caramel-app/tests/unit/sources-page.test.ts @@ -0,0 +1,75 @@ +import SourcesPage, { generateMetadata } from '@/app/(marketing)/sources/page' +import type { ReactElement } from 'react' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +// Pins (marketing)/sources/page.tsx's indexability: the table is server- +// rendered from listActiveSources, and with ZERO active sources the page is an +// empty shell (prod on 2026-09-11: `/api/sources` → [], GSC "crawled, not +// indexed"), so generateMetadata must noindex it — while keeping every other +// piece of the static metadata (title, canonical, OG) intact. The catalog read +// is mocked at the couponsRepo boundary; the client component is stubbed so +// importing the page doesn't drag the 'use client' tree into a node run. + +const { repoMock } = vi.hoisted(() => ({ + repoMock: { listActiveSources: vi.fn() }, +})) +vi.mock('@/lib/couponsRepo', () => repoMock) +vi.mock('@/app/(marketing)/sources/SourcesPageClient', () => ({ + default: () => null, +})) + +const ACTIVE_SOURCE = { + id: 'src-1', + source: 'Caramel Sample Feed A', + websites: ['ebay.com'], + status: 'ACTIVE', + total_coupons: 12, + total_used: 3, + total_expired: 1, +} + +beforeEach(() => { + repoMock.listActiveSources.mockReset() +}) + +describe('SourcesPage generateMetadata — empty-table noindex', () => { + it('with zero ACTIVE sources → robots noindex,follow; title/canonical/OG unchanged', async () => { + repoMock.listActiveSources.mockResolvedValue([]) + + const metadata = await generateMetadata() + + expect(metadata.robots).toEqual({ index: false, follow: true }) + expect(metadata.title).toBe( + 'Where Caramel Coupon Codes Come From | Sources', + ) + expect(metadata.alternates?.canonical).toBe('/sources') + expect(metadata.openGraph?.title).toBe( + 'Where Caramel Coupon Codes Come From | Sources', + ) + }) + + it('with at least one ACTIVE source → indexable (no robots override)', async () => { + repoMock.listActiveSources.mockResolvedValue([ACTIVE_SOURCE]) + + const metadata = await generateMetadata() + + expect(metadata.robots).toBeUndefined() + expect(metadata.alternates?.canonical).toBe('/sources') + }) +}) + +describe('SourcesPage body', () => { + it('server-renders the client table with the mapped source metrics from the same read', async () => { + repoMock.listActiveSources.mockResolvedValue([ACTIVE_SOURCE]) + + const el = (await SourcesPage()) as ReactElement<{ + initialSources: Array> + }> + + expect(el.props.initialSources).toHaveLength(1) + expect(el.props.initialSources[0]).toMatchObject({ + id: 'src-1', + source: 'Caramel Sample Feed A', + }) + }) +}) diff --git a/apps/caramel-app/tests/unit/storeIndexability.test.ts b/apps/caramel-app/tests/unit/storeIndexability.test.ts new file mode 100644 index 00000000..b3279fe2 --- /dev/null +++ b/apps/caramel-app/tests/unit/storeIndexability.test.ts @@ -0,0 +1,98 @@ +import { evaluateStorePageIndexability } from '@/lib/seo/storeIndexability' +import { resolveStoreDomain } from '@/lib/storeDomain' +import { describe, expect, it } from 'vitest' + +// Pins THE store-page indexability policy — the single function the sitemap +// (via sitemapStores.ts) and the page's generateMetadata both consult. Cases +// are the real prod shapes from the 2026-09-11 GSC audit +// (caramel-artifact/seo-2026-09-11/audit-findings.md §Sitemap), fed through +// the real resolveStoreDomain so the test proves the whole slug→verdict path. + +describe('evaluateStorePageIndexability — a real store with ≥1 visible coupon is indexable', () => { + it.each([ + ['gap.com', 41], + ['roborock.com', 16], + ['pandora.net', 58], + ['enasco.com', 3], + ])('%s with %d coupons → indexable, reason null', (base, count) => { + expect( + evaluateStorePageIndexability({ base, visibleCouponCount: count }), + ).toEqual({ indexable: true, reason: null }) + }) + + it('exactly ONE coupon is enough (a single live code is still a real answer)', () => { + expect( + evaluateStorePageIndexability({ + base: 'kickscrew.com', + visibleCouponCount: 1, + }), + ).toEqual({ indexable: true, reason: null }) + }) +}) + +describe('evaluateStorePageIndexability — zero visible coupons → no-coupons', () => { + it.each([0, -1, Number.NaN])( + 'count %s → not indexable, reason no-coupons', + count => { + expect( + evaluateStorePageIndexability({ + base: 'example.com', + visibleCouponCount: count, + }), + ).toEqual({ indexable: false, reason: 'no-coupons' }) + }, + ) +}) + +describe('evaluateStorePageIndexability — a slug that names no store → not-a-store', () => { + it.each([null, undefined, '', ' '])( + 'base %s → not indexable, reason not-a-store (even with coupons)', + base => { + expect( + evaluateStorePageIndexability({ + base, + visibleCouponCount: 50, + }), + ).toEqual({ indexable: false, reason: 'not-a-store' }) + }, + ) + + it.each([ + // The one non-domain slug the served sitemap carried. + 'dhl.com-us-en-home.html', + // Bare public suffixes — the 2026-08-05 "co.uk store" incident. + 'co.uk', + 'com.au', + // Reserved / unregistrable TLD (the e2e soft-404 probe slug). + 'no-coupons-here-zz.example', + ])( + 'resolveStoreDomain(%s) is null, so the policy says not-a-store', + slug => { + const base = resolveStoreDomain(slug) + expect(base).toBeNull() + expect( + evaluateStorePageIndexability({ + base, + visibleCouponCount: 12, + }), + ).toEqual({ indexable: false, reason: 'not-a-store' }) + }, + ) +}) + +describe('evaluateStorePageIndexability — the canonical base is what gets judged, never the raw slug', () => { + it.each([ + ['athleta.gap.com', 'gap.com'], + ['au.shein.com', 'shein.com'], + ['Brooklinen.com', 'brooklinen.com'], + ['eNasco.com', 'enasco.com'], + ['www.codecademy.com', 'codecademy.com'], + ])('%s resolves to %s and that base is indexable', (slug, expected) => { + const base = resolveStoreDomain(slug) + expect(base).toBe(expected) + expect( + evaluateStorePageIndexability({ base, visibleCouponCount: 2 }) + .indexable, + ).toBe(true) + }) +})