fix(seo): canonical store sitemap with lastmod behind one indexability policy

apps/caramel-app/e2e/seo-a11y.spec.ts

@@ -1,4 +1,9 @@
 import { expect, test } from '@playwright/test'
+// The app's own slug→registrable-domain rule (tldts / Public Suffix List).
+// Relative import on purpose: e2e collection runs in BOTH contexts (hermetic
+// and deployed, no generated prisma client) and this module is pure — no
+// `@/` alias, no prisma, no env.
+import { resolveStoreDomain } from '../src/lib/storeDomain'
 
 test.describe('SEO & Accessibility Basics', () => {
     test('home page has correct title', async ({ page }) => {
@@ -152,6 +157,52 @@ test.describe('Coupon pages — crawler-visible SEO', () => {
         const robots = await page.request.get('/robots.txt')
         expect(robots.status()).toBe(200)
     })
+
+    // Sitemap ↔ canonical agreement (GSC audit 2026-09-11: 84 subdomain slugs,
+    // 2 mixed-case slugs and 1 non-domain were listed whose own page
+    // canonicalized elsewhere, and 4,289 of 4,317 sitemap URLs were unknown to
+    // Google). Every store <loc> must BE the canonical the page emits — i.e.
+    // its own resolveStoreDomain, lowercase — and nothing may repeat. Holds
+    // on any non-empty catalog, so it stays ungated (two-context rule).
+    test('every store <loc> in sitemap.xml is its own canonical base (lowercase, PSL-resolved) and no <loc> repeats', async ({
+        page,
+    }) => {
+        const res = await page.request.get('/sitemap.xml')
+        expect(res.status()).toBe(200)
+        const xml = await res.text()
+
+        const locs = Array.from(xml.matchAll(/<loc>([^<]+)<\/loc>/g)).map(
+            m => m[1]!,
+        )
+        expect(locs.length).toBeGreaterThan(0)
+        expect(new Set(locs).size, 'duplicate <loc> in sitemap.xml').toBe(
+            locs.length,
+        )
+
+        const storeSlugs = locs
+            .map(loc => /\/coupons\/([^/?#]+)$/.exec(loc)?.[1])
+            .filter((slug): slug is string => Boolean(slug))
+            .map(slug => decodeURIComponent(slug))
+        // The catalog is never legitimately empty in either context.
+        expect(storeSlugs.length).toBeGreaterThan(0)
+
+        const offenders = storeSlugs.filter(
+            slug =>
+                slug !== slug.toLowerCase() ||
+                resolveStoreDomain(slug) !== slug,
+        )
+        expect(
+            offenders,
+            `store <loc>s that are not their own canonical base: ${offenders.slice(0, 20).join(', ')}`,
+        ).toEqual([])
+    })
+
+    test('/support is listed in sitemap.xml', async ({ baseURL, page }) => {
+        const res = await page.request.get('/sitemap.xml')
+        expect(res.status()).toBe(200)
+        const origin = (baseURL ?? '').replace(/\/+$/, '')
+        expect(await res.text()).toContain(`<loc>${origin}/support</loc>`)
+    })
 })
 
 test.describe('Responsive - Mobile Viewport', () => {

apps/caramel-app/e2e/seo-regression.spec.ts

@@ -28,6 +28,7 @@ import { expect, test } from '@playwright/test'
 //   /pricing           1453 / 1453   -> min 1000
 //   /sources            499 /  499   -> min  350
 //   /privacy           2780 / 2784   -> min 1900
+//   /support            547 (prod, 2026-09-11)   -> min 380
 // (/coupons is thin on purpose: the card grid is a client fetch; its server
 // HTML carries the shell copy + sidebar. If a route legitimately gains or
 // loses big copy, re-measure with the snippet in the PR that added this file
@@ -39,6 +40,7 @@ const ROUTES: ReadonlyArray<{ path: string; minVisibleChars: number }> = [
     { path: '/pricing', minVisibleChars: 1000 },
     { path: '/sources', minVisibleChars: 350 },
     { path: '/privacy', minVisibleChars: 1900 },
+    { path: '/support', minVisibleChars: 380 },
 ]
 
 // Same production-origin set as src/app/robots.ts (and next.config.mjs's
@@ -280,14 +282,40 @@ test.describe('SEO regression gate (raw server HTML)', () => {
             (xml.match(/<\/loc>/g) ?? []).length,
         )
 
-        // The 6 static marketing routes, emitted against the deployment's
+        // The static marketing routes, emitted against the deployment's
         // own origin (sitemap.ts builds each <loc> from BASE_URL, which
         // matches the origin this suite targets in all CI contexts).
+        //
+        // /sources is the one conditional entry: sitemap.ts lists it only
+        // when there is ≥1 ACTIVE source (otherwise the page is an empty
+        // shell that noindexes itself), so it is asserted against the SAME
+        // read the sitemap uses — /api/sources — rather than assumed. The
+        // hermetic seed has 2 ACTIVE sources; the deployed site may have 0.
         const origin = stripTrailingSlash(baseURL ?? '')
         for (const { path: routePath } of ROUTES) {
+            if (routePath === '/sources') continue
             const loc = `<loc>${origin}${routePath}</loc>`
             expect(xml, `sitemap.xml missing ${loc}`).toContain(loc)
         }
+
+        const sourcesRes = await page.request.get('/api/sources')
+        expect(sourcesRes.ok()).toBe(true)
+        const sourcesBody = (await sourcesRes.json()) as { data?: unknown }
+        const activeSources = Array.isArray(sourcesBody.data)
+            ? sourcesBody.data.length
+            : 0
+        const sourcesLoc = `<loc>${origin}/sources</loc>`
+        if (activeSources > 0) {
+            expect(
+                xml,
+                `${activeSources} ACTIVE source(s) but sitemap.xml omits ${sourcesLoc}`,
+            ).toContain(sourcesLoc)
+        } else {
+            expect(
+                xml,
+                `0 ACTIVE sources but sitemap.xml lists ${sourcesLoc}`,
+            ).not.toContain(sourcesLoc)
+        }
     })
 
     test('llms.txt is served for answer engines', async ({ page }) => {

apps/caramel-app/prisma/migrations/20260912120000_lowercase_coupon_sites/migration.sql

@@ -0,0 +1,19 @@
+-- Data-only migration: NO schema change (prisma migrate diff against the
+-- schema stays empty, so the schema-drift job is unaffected).
+--
+-- Domains are case-insensitive, but `coupons.site` was stored verbatim from
+-- the producer and the store-page reads match `site = $base OR site LIKE
+-- '%.' || $base` case-SENSITIVELY on a lowercased $base (resolveStoreDomain
+-- lowercases). Measured on prod 2026-09-11: 2 sites carried uppercase
+-- (`Brooklinen.com`, `eNasco.com`) — the eNasco rows were unreachable from
+-- ANY store page (/coupons/enasco.com and /coupons/eNasco.com both rendered
+-- the empty state with noindex) while the sitemap listed eNasco.com anyway.
+--
+-- From this migration on, applyCatalogRows lowercases `site` on every write,
+-- so this backfill is the one-time catch-up. Idempotent: the WHERE touches
+-- only rows that actually differ, so a re-run is a no-op. `updated_at` is
+-- deliberately NOT bumped — the ingest only-if-newer rule keys on it, and a
+-- bump would freeze these rows against the producer's next push.
+UPDATE "public"."coupons"
+SET site = lower(site)
+WHERE site IS NOT NULL AND site <> lower(site);

apps/caramel-app/src/app/(marketing)/coupons/[store]/page.tsx

@@ -5,6 +5,7 @@ import { attachSignals } from '@/lib/couponSignals'
 import { listStoreCoupons } from '@/lib/couponsRepo'
 import { BASE_URL } from '@/lib/env.client'
 import { jsonLdString } from '@/lib/jsonLd'
+import { evaluateStorePageIndexability } from '@/lib/seo/storeIndexability'
 import { resolveStoreDomain } from '@/lib/storeDomain'
 import type { Coupon } from '@/types/coupon'
 import type { Metadata } from 'next'
@@ -67,24 +68,39 @@ export async function generateMetadata({
     const { store } = await Promise.resolve(params)
     const storeParam = typeof store === 'string' ? safeDecode(store) : ''
     const base = getBaseDomain(storeParam)
-    if (!storeParam || !base) {
+    // ONE indexability policy, shared with the sitemap (app/sitemap.ts via
+    // src/lib/seo/sitemapStores.ts): a slug naming no registrable store, or a
+    // store with zero visible coupons, is `noindex, follow`, and the sitemap
+    // omits exactly those pages. fetchStoreCoupons short-circuits (no catalog
+    // read) when `base` is empty, and cache() shares the read with the body.
+    const { total } = await fetchStoreCoupons(storeParam)
+    const verdict = evaluateStorePageIndexability({
+        base,
+        visibleCouponCount: total,
+    })
+    // `follow` stays on in every noindex case: the links off the page (popular
+    // stores, header, footer) are still worth crawling.
+    const robots = verdict.indexable
+        ? undefined
+        : ({ index: false, follow: true } as const)
+
+    if (verdict.reason === 'not-a-store') {
         /* A slug that resolves to no registrable domain is not a store at all,
          * and this route still answers 200 for it (the body renders the honest
          * empty state rather than 404ing). That is the soft-404 bloat the
-         * zero-coupon rule below exists to keep out of the index — only more so,
+         * zero-coupon rule exists to keep out of the index — only more so,
          * because there is no store here to have coupons in the first place.
          *
          * It only became reachable when getBaseDomain moved to the Public Suffix
          * List: the old "last two labels" helper always returned SOMETHING, so
          * this branch was effectively dead and inherited no robots directive.
          * Caught by e2e/seo-a11y.spec.ts, which asks for /coupons/…-zz.example —
          * a slug the PSL correctly refuses, since `.example` is reserved and
-         * cannot be registered. `follow` stays on for the same reason it does
-         * below: the links off the page are still worth crawling. */
+         * cannot be registered. */
         return {
             title: 'Coupons | Caramel',
             description: 'Find coupons and promo codes on Caramel.',
-            robots: { index: false, follow: true },
+            robots,
         }
     }
     // Declaring `openGraph` below REPLACES the root layout's object wholesale
@@ -100,16 +116,15 @@ export async function generateMetadata({
     // base-domain URLs — this makes the page agree with it.)
     const canonical = `${baseUrl}/coupons/${encodeURIComponent(base)}`
     // Stores with zero visible coupons stay reachable (the prose section
-    // renders an honest empty state) but are noindexed: thousands of thin
-    // "no codes right now" pages in the index are soft-404 bloat. cache()
-    // makes this share one catalog read with the page body.
-    const { total } = await fetchStoreCoupons(storeParam)
+    // renders an honest empty state) but are noindexed (verdict.reason ===
+    // 'no-coupons'): thousands of thin "no codes right now" pages in the
+    // index are soft-404 bloat.
 
     return {
         title,
         description,
         alternates: { canonical },
-        robots: total === 0 ? { index: false, follow: true } : undefined,
+        robots,
         openGraph: {
             type: 'website',
             url: canonical,

apps/caramel-app/src/app/(marketing)/sources/page.tsx

@@ -2,6 +2,7 @@ import { listActiveSources } from '@/lib/couponsRepo'
 import { BASE_URL } from '@/lib/env.client'
 import { toSourceMetrics } from '@/lib/sourceMetrics'
 import type { Metadata } from 'next'
+import { cache } from 'react'
 import SourcesPageClient from './SourcesPageClient'
 
 // This page reads the coupon catalog from Postgres, and the production image
@@ -17,7 +18,7 @@ const base = BASE_URL
 const canonicalUrl = `${base}/sources`
 const banner = `${base}/caramel_banner.png`
 
-export const metadata: Metadata = {
+const baseMetadata: Metadata = {
     title,
     description,
     alternates: {
@@ -48,11 +49,29 @@ export const metadata: Metadata = {
     },
 }
 
+// cache(): generateMetadata needs the active-source list too (for the
+// empty-table noindex below), and React request-level caching makes that share
+// ONE catalog read with the page body — same pattern as the store page's
+// fetchStoreCoupons.
+const fetchActiveSources = cache(async () => listActiveSources())
+
+export async function generateMetadata(): Promise<Metadata> {
+    // With zero ACTIVE sources the page is an empty shell (prod had 0 on
+    // 2026-09-11 and GSC reported it crawled-not-indexed). Keep it reachable
+    // and followed, but out of the index until there is a table to index; the
+    // sitemap (app/sitemap.ts) lists /sources under the same condition.
+    const sources = await fetchActiveSources()
+    return {
+        ...baseMetadata,
+        robots: sources.length > 0 ? undefined : { index: false, follow: true },
+    }
+}
+
 export default async function SourcesPage() {
     // SEO: fetch the initial table server-side (same read + mapper as
     // /api/sources) so crawlers get the populated HTML instead of the old
     // client-fetch "Loading..." shell. The client keeps refetching through
     // the API after a source submission.
-    const initialSources = toSourceMetrics(await listActiveSources())
+    const initialSources = toSourceMetrics(await fetchActiveSources())
     return <SourcesPageClient initialSources={initialSources} />
 }

apps/caramel-app/src/app/sitemap.ts

@@ -1,54 +1,80 @@
-import { listStoreOptions } from '@/lib/couponsRepo'
+import { listActiveSources, listStoreSitemapEntries } from '@/lib/couponsRepo'
 import { BASE_URL } from '@/lib/env.client'
+import { collapseStoreRows } from '@/lib/seo/sitemapStores'
 import type { MetadataRoute } from 'next'
 
 // The store half of this sitemap reads the coupon catalog from Postgres, and
 // the production image builds against a deliberately unreachable placeholder
 // DATABASE_URL (see the Dockerfile's `.invalid` builder env) — so this route
 // must be rendered per-request, never prerendered at build time. Crawlers hit
-// it rarely and the read is a single indexed DISTINCT, so per-request is cheap.
+// it rarely and the read is a single indexed GROUP BY, so per-request is cheap.
 export const dynamic = 'force-dynamic'
 
 const origin = BASE_URL.replace(/\/+$/, '')
 
-// Upper bound on `/coupons/[store]` entries. The sitemap spec caps a single
-// file at 50,000 URLs; this stays well under it and bounds the query. If the
-// catalog ever outgrows it, the fix is a sitemap index, not a bigger number.
+// Upper bound on grouped `coupons.site` rows feeding `/coupons/[store]`
+// entries. The sitemap spec caps a single file at 50,000 URLs; this stays well
+// under it and bounds the query. If the catalog ever outgrows it, the fix is a
+// sitemap index, not a bigger number.
 const STORE_URL_LIMIT = 5000
 
-// Public marketing routes. Auth pages ((auth)/login, signup, verify) and
-// /profile are deliberately absent — they are disallowed in robots.ts.
-const STATIC_ROUTES: ReadonlyArray<{
+type StaticRoute = {
     path: string
     changeFrequency: MetadataRoute.Sitemap[number]['changeFrequency']
     priority: number
-}> = [
+}
+
+// Public marketing routes. Auth pages ((auth)/login, signup, verify) and
+// /profile are deliberately absent — they are disallowed in robots.ts.
+// /support is indexable and header-linked, so it belongs here (it was missing
+// until 2026-09; GSC saw it only through links).
+const STATIC_ROUTES: ReadonlyArray<StaticRoute> = [
     { path: '/', changeFrequency: 'weekly', priority: 1 },
     { path: '/coupons', changeFrequency: 'daily', priority: 0.9 },
     { path: '/supported-stores', changeFrequency: 'weekly', priority: 0.8 },
     { path: '/pricing', changeFrequency: 'monthly', priority: 0.7 },
-    { path: '/sources', changeFrequency: 'weekly', priority: 0.6 },
+    { path: '/support', changeFrequency: 'monthly', priority: 0.5 },
     { path: '/privacy', changeFrequency: 'yearly', priority: 0.3 },
 ]
 
+// /sources renders a table of ACTIVE sources. With none (prod had 0 on
+// 2026-09-11 — `/api/sources` returned `[]`) it is an empty shell that the page
+// itself noindexes ((marketing)/sources/page.tsx), so it is listed only when
+// there is something to index. Same read the page uses.
+const SOURCES_ROUTE: StaticRoute = {
+    path: '/sources',
+    changeFrequency: 'weekly',
+    priority: 0.6,
+}
+
 export default async function sitemap(): Promise<MetadataRoute.Sitemap> {
-    // Same read the /api/coupons/stores autocomplete uses: DISTINCT visible
-    // sites, empty query = no ILIKE filter. No `lastModified` is emitted for
-    // store pages because this row shape carries no timestamp — an invented
-    // date is worse than none.
-    const storeRows = await listStoreOptions('', STORE_URL_LIMIT)
-    const stores = storeRows
-        .map(row => row.site)
-        .filter((site): site is string => Boolean(site && site.trim()))
+    // One aggregate row per raw `coupons.site` (visible coupons only), then
+    // collapsed to the CANONICAL registrable domain the page canonicalizes to
+    // and filtered by the SAME indexability policy the page's robots meta
+    // uses (src/lib/seo/storeIndexability.ts) — so every store <loc> here is
+    // its own canonical and never a noindexed page. `lastModified` is the
+    // newest `coupons.updated_at` folded into that base: a real catalog
+    // timestamp, the freshness signal Google needs to re-read a sitemap.
+    const [storeRows, activeSources] = await Promise.all([
+        listStoreSitemapEntries(STORE_URL_LIMIT),
+        listActiveSources(),
+    ])
+    const stores = collapseStoreRows(storeRows)
+
+    const staticRoutes: StaticRoute[] =
+        activeSources.length > 0
+            ? [...STATIC_ROUTES, SOURCES_ROUTE]
+            : [...STATIC_ROUTES]
 
     return [
-        ...STATIC_ROUTES.map(route => ({
+        ...staticRoutes.map(route => ({
             url: `${origin}${route.path}`,
             changeFrequency: route.changeFrequency,
             priority: route.priority,
         })),
-        ...stores.map(site => ({
-            url: `${origin}/coupons/${encodeURIComponent(site)}`,
+        ...stores.map(store => ({
+            url: `${origin}/coupons/${encodeURIComponent(store.base)}`,
+            ...(store.lastModified ? { lastModified: store.lastModified } : {}),
             changeFrequency: 'daily' as const,
             priority: 0.7,
         })),

apps/caramel-app/src/lib/catalog/applyCatalogRows.ts

@@ -102,6 +102,21 @@ class TombstoneGateError extends Error {
     }
 }
 
+/**
+ * The ONE write-side normalization in this engine: `site` is stored lowercase.
+ * Domains are case-insensitive, but every store read matches
+ * `site = $base OR site LIKE '%.' || $base` case-SENSITIVELY (plain equality
+ * keeps `coupons_site_idx` usable) against a base resolveStoreDomain has
+ * lowercased — so a mixed-case producer value was unreachable from its own
+ * canonical page (prod 2026-09-11: `eNasco.com`, `Brooklinen.com`). The
+ * `lowercase_coupon_sites` migration backfilled existing rows; this keeps new
+ * ones honest. Everything else (status, discount_type, code) stays RAW per
+ * ingestSchemas.ts — this is a domain-name identity, not a vocabulary.
+ */
+function normalizeSite(site: string | null): string | null {
+    return site == null ? null : site.toLowerCase()
+}
+
 function chunk<T>(items: readonly T[], size: number): T[][] {
     const out: T[][] = []
     for (let i = 0; i < items.length; i += size) {
@@ -339,7 +354,7 @@ export async function applyCatalogRows(
                     for (const rowChunk of chunk(toWrite, WRITE_CHUNK)) {
                         const tuples = rowChunk.map(
                             r =>
-                                Prisma.sql`(${r.id}, ${r.code}, ${r.site}, ${r.title}, ${r.description}, ${r.rating}, ${r.discount_type}, ${r.discount_amount}, ${r.expiry}, ${r.expired}, ${r.times_used}, ${r.last_time_used ?? null}, ${r.status}, ${r.verification_message}, ${r.created_at ?? new Date()}, ${r.updated_at})`,
+                                Prisma.sql`(${r.id}, ${r.code}, ${normalizeSite(r.site)}, ${r.title}, ${r.description}, ${r.rating}, ${r.discount_type}, ${r.discount_amount}, ${r.expiry}, ${r.expired}, ${r.times_used}, ${r.last_time_used ?? null}, ${r.status}, ${r.verification_message}, ${r.created_at ?? new Date()}, ${r.updated_at})`,
                         )
                         await tx.$executeRaw(
                             Prisma.sql`