diff --git a/next.config.ts b/next.config.ts index 465eea7..821007f 100644 --- a/next.config.ts +++ b/next.config.ts @@ -38,10 +38,12 @@ const nextConfig: NextConfig = { { protocol: "https", hostname: "www.centrecanada.org" }, { protocol: "https", hostname: "pirs.bc.ca" }, { protocol: "https", hostname: "www.pirs.bc.ca" }, - // Phase-2 crawler source, staged disabled in events-crawler/index.ts. Allowlisted + // Phase-2 crawler sources, staged disabled in events-crawler/index.ts. Allowlisted // ahead of activation so flipping `enabled` needs no web-side change. { protocol: "https", hostname: "westvanlibrary.ca" }, { protocol: "https", hostname: "www.westvanlibrary.ca" }, + // BiblioCommons serves each library's event covers off its own tenant subdomain. + { protocol: "https", hostname: "vpl.bibliocommons.com" }, // events-crawler tier-2 covers (Pexels stock photos for events with no source image) { protocol: "https", hostname: "images.pexels.com" }, // News article images diff --git a/supabase/functions/events-crawler/adapters/bibliocommons.ts b/supabase/functions/events-crawler/adapters/bibliocommons.ts new file mode 100644 index 0000000..4085bd2 --- /dev/null +++ b/supabase/functions/events-crawler/adapters/bibliocommons.ts @@ -0,0 +1,271 @@ +// Adapter: BiblioCommons BiblioEvents. Currently one source — Vancouver Public Library. +// +// The gateway is the library SPA's own JSON API: unauthenticated, but undocumented and +// unversioned in practice, so treat a shape change as expected maintenance rather than a +// surprise. Every failure path returns [] so one library can't take a run down. +// +// CAUTION when adding a library: the `host` here is a BiblioCommons tenant slug, and the +// obvious guess is not always the right tenant. `bpl` is BOSTON Public Library, not +// Burnaby — it returns a perfectly healthy feed of Copley Square events that would look +// plausible in a log and be entirely wrong in the app. Burnaby's tenant is `burnaby`, and +// it answers "The Events feature is not available", so it cannot be crawled this way at +// all. Preview any new slug with dryrun.ts and read the branch names before enabling it. +// +// The awkward part is that the API supports NO date filtering and NO sorting. Verified +// 2026-07-31 against VPL: startDate, endDate, from/to, start/end, minDate, after, +// dateRange, sort and sortBy are all silently ignored — `count` stays at the full catalog +// size and the returned order is arbitrary (page 1 came back 2026-08-20, 2026-12-01, +// 2026-10-31). `limit` caps out at 100; 500 and 1000 return an empty body. +// +// So "the soonest 25 events inside the 4-month window" cannot be expressed as a query. We +// page the whole catalog (VPL: 1,997 events over 20 pages), filter and sort client-side, +// then cap. Fetching only the first few pages would be far cheaper but would return an +// arbitrary subset rather than the soonest events, with no way to tell from the result +// that it had happened. +// +// Images are resolved only for the final capped slice, not for every candidate — the +// three-tier resolver does a HEAD probe per image, and probing thousands would dominate +// the run. + +import { FETCH_TIMEOUT_MS, MAX_PER_ORG, MAX_TITLE_CHARS, USER_AGENT } from '../lib/constants.ts'; +import { offsetIsoToUtc } from '../lib/dates.ts'; +import { genreForEvent } from '../lib/genre.ts'; +import { resolveCover } from '../lib/images.ts'; +import { isSettlementRelevant } from '../lib/relevance.ts'; +import { clean, htmlToParagraphs, isOnlineVenueName } from '../lib/text.ts'; +import type { AdapterContext, EventRow, Source } from '../lib/types.ts'; + +const GATEWAY = 'https://gateway.bibliocommons.com/v2/libraries'; +/** The API's own ceiling — 500 and 1000 return an empty body. */ +const PAGE_LIMIT = 100; +/** + * Backstop against a catalog that grows (or a pagination bug that never terminates). + * Comfortably above today's worst case — VPL at 20 pages — with room for a busier tenant. + * Hitting it is logged, never silent: a truncated crawl must not look like a complete one. + */ +const MAX_PAGES = 45; +/** Concurrent page fetches. Enough to keep the run short, polite enough not to hammer. */ +const PAGE_CONCURRENCY = 4; + +interface BiblioAddress { + number?: string; + street?: string; + city?: string; + state?: string; + zip?: string; +} + +interface BiblioEntities { + events?: Record; + locations?: Record; + places?: Record; + images?: Record; +} + +interface BiblioEvent { + id?: string; + /** UTC, ISO with Z. `definition.start` is the same instant without an offset — don't use it. */ + indexStart?: string; + indexEnd?: string; + definition?: { + title?: string; + description?: string; + isCancelled?: boolean; + featuredImageId?: string | null; + branchLocationId?: string | null; + nonBranchLocationId?: string | null; + contact?: { name?: string }; + }; +} + +interface BiblioPage { + events?: { pagination?: { pages?: number; count?: number } }; + entities?: BiblioEntities; +} + +/** One event as parsed, before the expensive image tier runs. */ +interface Candidate { + id: string; + title: string; + startIso: string; + endIso: string | null; + description: string | null; + location: string; + address: string | null; + eventType: EventRow['event_type']; + imageUrl: string | null; +} + +async function fetchPage(source: Source, page: number): Promise { + const url = `${GATEWAY}/${encodeURIComponent(source.host)}/events?limit=${PAGE_LIMIT}&page=${page}`; + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), FETCH_TIMEOUT_MS); + try { + const res = await fetch(url, { + signal: controller.signal, + headers: { 'User-Agent': USER_AGENT, Accept: 'application/json' }, + }); + if (!res.ok) { + console.error(`events-crawler: ${source.slug} page ${page} returned HTTP ${res.status}`); + return null; + } + return (await res.json()) as BiblioPage; + } catch (error) { + console.error(`events-crawler: ${source.slug} page ${page} failed:`, error); + return null; + } finally { + clearTimeout(timer); + } +} + +/** + * Parse one page into candidates. The locations / places / images maps are per-response, + * so they must be read from the same page as the events that reference them. + */ +function candidatesFromPage(page: BiblioPage, source: Source, ctx: AdapterContext): Candidate[] { + const entities = page.entities ?? {}; + const events = entities.events ?? {}; + const out: Candidate[] = []; + + for (const [id, ev] of Object.entries(events)) { + const def = ev?.definition; + if (!def || def.isCancelled === true) continue; + + // Filter on the FULL cleaned title, then truncate for storage — a relevant keyword + // past MAX_TITLE_CHARS would otherwise be invisible to the filter. + const fullTitle = clean(def.title); + if (!fullTitle) continue; + if (source.relevanceFilter && !isSettlementRelevant(fullTitle)) continue; + const title = fullTitle.slice(0, MAX_TITLE_CHARS); + + const startIso = offsetIsoToUtc(ev.indexStart); + if (!startIso) continue; + // Both edges enforced here: the API has no "from today" bound, so without the lower + // check the catalog's past events would be ingested alongside the upcoming ones. + const startMs = Date.parse(startIso); + if (startMs < ctx.nowMs || startMs > ctx.windowEndMs) continue; + + const branch = def.branchLocationId ? entities.locations?.[def.branchLocationId] : undefined; + const place = def.nonBranchLocationId + ? entities.places?.[def.nonBranchLocationId] + : undefined; + const location = + clean(branch?.name) || clean(place?.name) || clean(def.contact?.name); + if (!location) continue; // events.location is NOT NULL + + const addr = place?.address; + const address = addr + ? [[addr.number, addr.street].map((p) => clean(p)).filter(Boolean).join(' '), addr.city, addr.state, addr.zip] + .map((p) => clean(p)) + .filter(Boolean) + .join(', ') || null + : null; + + // A featured image tagged EventType is the shared category placeholder (the same + // "activities-and-games.png" is reused across dozens of unrelated events), so skip it + // and let the Pexels tier supply something that actually matches this event. + const image = def.featuredImageId ? entities.images?.[def.featuredImageId] : undefined; + const imageUrl = + image && image.tag !== 'EventType' && typeof image.url === 'string' ? image.url : null; + + out.push({ + id: clean(ev.id) || id, + title, + startIso, + endIso: offsetIsoToUtc(ev.indexEnd), + description: htmlToParagraphs(def.description) || null, + location, + address, + eventType: isOnlineVenueName(location) ? 'online' : 'in-person', + imageUrl, + }); + } + return out; +} + +export async function fetchEvents(source: Source, ctx: AdapterContext): Promise { + const first = await fetchPage(source, 1); + if (!first) return []; + + const reported = first.events?.pagination?.pages ?? 1; + const totalPages = Math.max(1, Math.min(reported, MAX_PAGES)); + if (reported > MAX_PAGES) { + console.warn( + `events-crawler: ${source.slug} reports ${reported} pages, capped at ${MAX_PAGES} — ` + + `results are a partial view of the catalog; raise MAX_PAGES.`, + ); + } + + const candidates: Candidate[] = candidatesFromPage(first, source, ctx); + + // Bounded concurrency: a simple sliding window over the remaining page numbers. + const remaining: number[] = []; + for (let p = 2; p <= totalPages; p++) remaining.push(p); + let failedPages = 0; + for (let i = 0; i < remaining.length; i += PAGE_CONCURRENCY) { + const batch = remaining.slice(i, i + PAGE_CONCURRENCY); + const pages = await Promise.all(batch.map((p) => fetchPage(source, p))); + for (const page of pages) { + if (page) { + candidates.push(...candidatesFromPage(page, source, ctx)); + } else { + failedPages++; + } + } + } + // A failed page silently shrinks the candidate pool, which would otherwise be + // indistinguishable from a complete crawl in the summary below — same reasoning as the + // MAX_PAGES warning. + if (failedPages > 0) { + console.warn( + `events-crawler: ${source.slug} had ${failedPages} failed page fetch(es) of ` + + `${totalPages} — results are a partial view of the catalog.`, + ); + } + + // Dedupe by event id before sorting: the catalog has no stable order and is read over + // many round trips, so a catalog shift mid-pagination can surface the same event on two + // pages. A duplicate would otherwise consume one of the MAX_PER_ORG slots and pay for a + // second, redundant cover probe. + const uniqueById = new Map(); + for (const c of candidates) uniqueById.set(c.id, c); + const deduped = [...uniqueById.values()]; + + // Sort ascending and cap only after the whole catalog is in hand — the API's arbitrary + // order means any earlier cut would be an arbitrary subset, not the soonest events. + deduped.sort((a, b) => a.startIso.localeCompare(b.startIso)); + const selected = deduped.slice(0, MAX_PER_ORG); + + console.log( + `events-crawler: ${source.slug} scanned ${totalPages} page(s), ` + + `${candidates.length} in-window${source.relevanceFilter ? ' + relevant' : ''} ` + + `(${deduped.length} unique), taking soonest ${selected.length}`, + ); + + // allSettled so one failed cover lookup can't reject the batch and discard the source. + const settled = await Promise.allSettled( + selected.map(async (c): Promise => { + const externalLink = `https://${source.host}.bibliocommons.com/events/${c.id}`; + return { + title: c.title, + description: c.description, + event_datetime: c.startIso, + event_end_datetime: c.endIso, + location: c.location, + event_type: c.eventType, + cover_photo_url: await resolveCover(c.imageUrl, c.title, externalLink, ctx.pexelsCache), + external_link: externalLink, + hosted_by: source.name, + address: c.address, + genre: genreForEvent(c.title, c.description), + source: `crawler:${source.slug}`, + }; + }), + ); + const rows: EventRow[] = []; + for (const result of settled) { + if (result.status === 'fulfilled') rows.push(result.value); + else console.error(`events-crawler: ${source.slug} row failed:`, result.reason); + } + return rows; +} diff --git a/supabase/functions/events-crawler/adapters/tribe.ts b/supabase/functions/events-crawler/adapters/tribe.ts new file mode 100644 index 0000000..67ed539 --- /dev/null +++ b/supabase/functions/events-crawler/adapters/tribe.ts @@ -0,0 +1,193 @@ +// Adapter: WordPress "The Events Calendar" (Tribe) JSON REST API. +// +// Serves every Phase-1 settlement org plus West Vancouver Memorial Library. Extracted +// verbatim from the pre-adapter crawler — behaviour is unchanged apart from the shared +// relevance filter, which only applies to sources that opt in. + +import { + FETCH_TIMEOUT_MS, + MAX_PER_ORG, + MAX_TITLE_CHARS, + USER_AGENT, +} from '../lib/constants.ts'; +import { toIsoUtc } from '../lib/dates.ts'; +import { genreForEvent } from '../lib/genre.ts'; +import { resolveCover } from '../lib/images.ts'; +import { isSettlementRelevant } from '../lib/relevance.ts'; +import { clean, htmlToParagraphs, isOnlineVenueName } from '../lib/text.ts'; +import type { AdapterContext, EventRow, Source } from '../lib/types.ts'; + +interface TribeVenue { + venue?: string; + address?: string; + city?: string; + stateprovince?: string; + province?: string; + zip?: string; +} + +interface TribeEvent { + status?: string; + hide_from_listings?: boolean; + title?: string; + url?: string; + utc_start_date?: string; + utc_end_date?: string; + excerpt?: string; + description?: string; + /** Tribe sends `[]` rather than null when an event has no venue. */ + venue?: TribeVenue | TribeVenue[] | null; + is_virtual?: boolean; + image?: { url?: string } | null; + organizer?: unknown; +} + +function organizerName(organizer: unknown): string | null { + if (!Array.isArray(organizer) || organizer.length === 0) return null; + const first: unknown = organizer[0]; + if (typeof first === 'string') return clean(first) || null; + if (first && typeof first === 'object' && 'organizer' in first) { + const value = (first as { organizer?: unknown }).organizer; + if (typeof value === 'string') return clean(value) || null; + } + return null; +} + +/** + * Map one Tribe REST event to an EventRow, or null when it can't be shaped into a valid + * row (missing NOT-NULL data, undeterminable location, past, hidden, filtered out, or + * starting beyond the rolling window). + */ +async function tribeEventToRow( + ev: TribeEvent | null, + source: Source, + ctx: AdapterContext, +): Promise { + if (!ev || typeof ev !== 'object') return null; + if (ev.status && ev.status !== 'publish') return null; + if (ev.hide_from_listings === true) return null; + + // Filter on the FULL cleaned title, then truncate for storage — a relevant keyword + // sitting past MAX_TITLE_CHARS would otherwise be invisible to the filter and the event + // silently dropped. + const fullTitle = clean(ev.title); + const title = fullTitle.slice(0, MAX_TITLE_CHARS); + const externalLink = typeof ev.url === 'string' ? ev.url.trim() : ''; + const eventDatetime = toIsoUtc(ev.utc_start_date); + if (!title || !externalLink || !eventDatetime) return null; // NOT NULL columns + + // Cheap rejections first, so a filtered or far-future event costs no HEAD probe or + // Pexels lookup. Backstop for the server-side ?end_date — see fetchEvents. + if (source.relevanceFilter && !isSettlementRelevant(fullTitle)) return null; + if (Date.parse(eventDatetime) > ctx.windowEndMs) return null; + + // location + event_type from the venue NAME first, then venue presence, then the + // virtual flag — see isOnlineVenueName for why the name has to win. + const rawVenue = ev.venue; + const venue = + rawVenue && typeof rawVenue === 'object' && !Array.isArray(rawVenue) ? rawVenue : null; + const venueName = venue ? clean(venue.venue) : ''; + const hasVenue = venueName.length > 0; + const isVirtual = ev.is_virtual === true; + + let location: string; + let eventType: EventRow['event_type']; + if (hasVenue && isOnlineVenueName(venueName)) { + location = venueName; // keep the source wording ('Online' / 'Webinar') + eventType = 'online'; + } else if (hasVenue) { + location = venueName; + eventType = isVirtual ? 'hybrid' : 'in-person'; + } else if (isVirtual) { + location = 'Online'; + eventType = 'online'; + } else { + return null; // neither a venue nor a virtual flag → no reliable location + } + + // description: prefer the (usually cleaner) excerpt, fall back to the body. + const excerpt = htmlToParagraphs(ev.excerpt); + const body = htmlToParagraphs(ev.description); + const description = (excerpt.length >= 40 ? excerpt : body) || null; + + const address = + venue && clean(venue.address) + ? [venue.address, venue.city, venue.stateprovince || venue.province, venue.zip] + .map((p) => clean(p)) + .filter(Boolean) + .join(', ') + : null; + + const imageUrl = ev.image && typeof ev.image.url === 'string' ? ev.image.url : null; + + return { + title, + description, + event_datetime: eventDatetime, + event_end_datetime: toIsoUtc(ev.utc_end_date), + location, + event_type: eventType, + cover_photo_url: await resolveCover(imageUrl, title, externalLink, ctx.pexelsCache), + external_link: externalLink, + hosted_by: organizerName(ev.organizer) ?? source.name, + address, + genre: genreForEvent(title, description), + source: `crawler:${source.slug}`, + }; +} + +export async function fetchEvents( + source: Source, + ctx: AdapterContext, +): Promise { + // ?end_date is a server-side bound; tribeEventToRow re-checks per row because a source + // whose API ignores the param would otherwise slip events years out past it. + const url = + `https://${source.host}/wp-json/tribe/events/v1/events` + + `?per_page=${MAX_PER_ORG}&start_date=${ctx.today}%2000:00:00` + + `&end_date=${ctx.windowEnd}%2023:59:59`; + + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), FETCH_TIMEOUT_MS); + try { + const res = await fetch(url, { + signal: controller.signal, + headers: { 'User-Agent': USER_AGENT, Accept: 'application/json' }, + }); + if (!res.ok) { + console.error(`events-crawler: ${source.slug} returned HTTP ${res.status}`); + return []; + } + const data: unknown = await res.json(); + const events: TribeEvent[] = + data && typeof data === 'object' && Array.isArray((data as { events?: unknown }).events) + ? ((data as { events: TribeEvent[] }).events) + : []; + // allSettled, not all: tribeEventToRow awaits network work (the cover HEAD probe and + // the Pexels lookup). With Promise.all a single rejection would reject the whole + // batch, hit the outer catch and return [] — one transient image failure discarding + // every event of this source for the entire weekly run. + const settled = await Promise.allSettled( + events.slice(0, MAX_PER_ORG).map((ev) => tribeEventToRow(ev, source, ctx)), + ); + const kept: EventRow[] = []; + for (const result of settled) { + if (result.status === 'fulfilled') { + if (result.value) kept.push(result.value); + } else { + console.error(`events-crawler: ${source.slug} row failed:`, result.reason); + } + } + if (source.relevanceFilter) { + console.log( + `events-crawler: ${source.slug} relevance filter kept ${kept.length} of ${events.length} fetched`, + ); + } + return kept; + } catch (error) { + console.error(`events-crawler: failed to fetch ${source.slug}:`, error); + return []; + } finally { + clearTimeout(timer); + } +} diff --git a/supabase/functions/events-crawler/dryrun.ts b/supabase/functions/events-crawler/dryrun.ts new file mode 100644 index 0000000..f182fe7 --- /dev/null +++ b/supabase/functions/events-crawler/dryrun.ts @@ -0,0 +1,86 @@ +// Read-only preview harness for the events-crawler adapters. +// +// SAFETY: this script has NO write path. It never imports the Supabase client, never +// constructs one, and has no --apply flag to add one — it fetches a source's feed and +// prints the rows the crawler WOULD insert. It cannot touch the shared events table, and +// it does not go near the cron. That is the whole point: the deployed function is +// INSERT-only with no dry-run mode, so invoking it to "test" a change writes to prod. +// +// It imports the real registry, adapter map and context factory from lib/sources.ts — +// the exact wiring index.ts uses — so the preview cannot drift from a real run. +// +// RUN (from web-app/): +// deno run --allow-net --allow-env=PEXELS_API_KEY \ +// supabase/functions/events-crawler/dryrun.ts --source vpl +// …--source vpl --json # full rows as JSON +// …--list # the registry, with enabled / filtered flags +// +// The env grant is scoped to PEXELS_API_KEY because the tier-2 cover lookup reads it. +// Leaving it unset locally is fine and is the common case: fetchPexelsCandidates returns +// [] without a key, so covers fall through to the deterministic Unsplash pool — which is +// exactly what production does when the secret is absent. +// +// Disabled sources can be previewed here — that is what makes it possible to review a +// staged source before anyone flips `enabled`. + +import { ADAPTERS, makeContext, SOURCES } from './lib/sources.ts'; +import type { EventRow } from './lib/types.ts'; + +function arg(name: string): string | undefined { + const i = Deno.args.indexOf(`--${name}`); + return i >= 0 ? Deno.args[i + 1] : undefined; +} + +function printRow(row: EventRow, i: number): void { + const end = row.event_end_datetime ? ` → ${row.event_end_datetime}` : ''; + console.log(`\n${String(i + 1).padStart(2)}. ${row.title}`); + console.log(` when ${row.event_datetime}${end}`); + console.log(` where ${row.location} [${row.event_type}]`); + if (row.address) console.log(` address ${row.address}`); + console.log(` genre ${row.genre}`); + console.log(` host ${row.hosted_by ?? '(none)'}`); + console.log(` link ${row.external_link}`); + console.log(` cover ${row.cover_photo_url ?? '(none)'}`); + const desc = (row.description ?? '').split('\n')[0] ?? ''; + console.log(` desc ${desc.slice(0, 100)}${desc.length > 100 ? '…' : ''}`); +} + +if (Deno.args.includes('--list')) { + console.log('Sources:'); + for (const s of SOURCES) { + const flags = [s.enabled ? 'enabled' : 'DISABLED', s.relevanceFilter ? 'relevance-filtered' : ''] + .filter(Boolean) + .join(', '); + console.log(` ${s.slug.padEnd(18)} ${s.kind.padEnd(15)} ${flags}`); + } + Deno.exit(0); +} + +const slug = arg('source'); +if (!slug) { + console.error('Usage: dryrun.ts --source [--json] | --list'); + Deno.exit(1); +} + +const source = SOURCES.find((s) => s.slug === slug); +if (!source) { + console.error(`Unknown source '${slug}'. Run with --list to see the registry.`); + Deno.exit(1); +} + +const ctx = makeContext(); + +console.log( + `DRY RUN — ${source.slug} (${source.kind}), window ${ctx.today} → ${ctx.windowEnd}, ` + + `${source.enabled ? 'enabled' : 'DISABLED in production'}. No writes.\n`, +); + +const rows = await ADAPTERS[source.kind](source, ctx); + +if (Deno.args.includes('--json')) { + console.log(JSON.stringify(rows, null, 2)); +} else { + rows.forEach(printRow); +} + +console.log(`\n${rows.length} row(s) would be inserted. Nothing was written.`); diff --git a/supabase/functions/events-crawler/index.ts b/supabase/functions/events-crawler/index.ts index f984598..18469ab 100644 --- a/supabase/functions/events-crawler/index.ts +++ b/supabase/functions/events-crawler/index.ts @@ -1,633 +1,32 @@ -// @ts-nocheck Deno runtime — Supabase Edge Functions (not part of the Next build) import 'jsr:@supabase/functions-js/edge-runtime.d.ts'; import { createClient } from 'jsr:@supabase/supabase-js@2'; -import { isPublicHttpUrl } from '../_shared/ssrf.ts'; -import { fetchPexelsCandidates } from '../_shared/pexels.ts'; + +import { ACTIVE_SOURCES, ADAPTERS, makeContext } from './lib/sources.ts'; +import type { EventRow } from './lib/types.ts'; // ============================================================================ -// events-crawler — BC settlement-org events → public.events +// events-crawler — BC settlement-org and library events → public.events // ---------------------------------------------------------------------------- // Cron-triggered (see 20260722120000_events_crawler.sql for the source column and -// 20260724120000_events_crawler_cron.sql for the schedule): pg_cron POSTs here -// with the service-role key as the Bearer token. The function pulls upcoming -// events from WordPress "The Events Calendar" (Tribe) JSON REST APIs and inserts -// new rows into public.events, shaped IDENTICALLY to the events Savar enters by -// hand (same columns, same CommunityEvent render path in EventCard.tsx + the -// event detail page). Scraped rows are tagged `source = 'crawler:'` so they -// stay distinguishable from manual rows. +// 20260724120000_events_crawler_cron.sql for the schedule): pg_cron POSTs here with the +// service-role key as the Bearer token. Each enabled source is pulled by its adapter and +// mapped to rows shaped IDENTICALLY to the events Savar enters by hand (same columns, +// same CommunityEvent render path in EventCard.tsx + the event detail page). Scraped rows +// are tagged `source = 'crawler:'` so they stay distinguishable from manual rows. // -// Mirrors supabase/functions/news-crawler/index.ts: same service-role Bearer -// auth (verify_jwt=true, no CORS — server-to-server only), the same image-quality -// trio (icon-URL reject + SSRF guard + HEAD size check), and the same -// deterministic Unsplash fallback pool. INSERT-only: no delete/prune. Rows outside -// the WINDOW_MONTHS rolling window are never ingested, and ones that age out of it -// simply stop matching the web reader's window filter — they are not removed, -// because mobile reads this same table with no date filter and a Past tab. +// Mirrors supabase/functions/news-crawler/index.ts: same service-role Bearer auth +// (verify_jwt=true, no CORS — server-to-server only), and the same image-quality trio +// (icon-URL reject + SSRF guard + HEAD size check). INSERT-only: no delete/prune. Rows +// outside the rolling window are never ingested, and ones that age out of it simply stop +// matching the web reader's window filter — they are not removed, because mobile reads +// this same table with no date filter and a Past tab. // -// Each row is tagged with a `genre` (see GENRE_RULES) so both apps can filter the -// tab by topic. +// Each row is tagged with a `genre` (lib/genre.ts) so both apps can filter by topic. // -// Shared prod DB (web + mobile) — inserted events appear in BOTH apps immediately. +// Shared prod DB (web + mobile) — inserted events appear in BOTH apps immediately, which +// is why new sources land `enabled: false`. See Source.enabled in lib/types.ts. // ============================================================================ -interface Org { - /** Stored into source as `crawler:`. */ - slug: string; - /** Fallback hosted_by / display name when the event has no organizer. */ - name: string; - /** WordPress host serving /wp-json/tribe/events/v1/events. */ - host: string; - /** - * Whether a run actually crawls this org. New orgs land as `false`. - * - * The weekly cron is LIVE (20260724120000_events_crawler_cron.sql — pg_cron jobid 18, - * 'events-crawler-weekly', Mondays 14:00 UTC), so deploying this function is what puts a - * new org into production: the next run would auto-publish it into the shared events - * table the mobile app reads. Landing new orgs disabled decouples the two, leaving - * activation as its own one-line, reviewable change — which needs Savar's sign-off, - * since it writes into shared mobile-facing data (CLAUDE.md, "Decision Authority"). - */ - enabled: boolean; -} - -// Phase 1 (enabled): the five highest-relevance orgs, all on The Events Calendar (Tribe). -// Phase 2 (staged, disabled): further sources from the 2026-07-29 scoping round. -// -// Add an org here (test /wp-json/tribe/events/v1/events first) to broaden coverage; -// remember to allowlist its image host in next.config.ts, and to land it `enabled: false`. -// -// NOTE on virtual events: none of these orgs sets Tribe's `is_virtual` flag (it ships -// with the paid Virtual Events add-on) — it is false or absent on every event. They mark -// online events by registering a venue named "Online" / "Webinar" instead, which is why -// isOnlineVenueName drives event_type. Don't rely on `is_virtual` alone. -// -// centrecanada.org is healthy but currently returns total:0 (empty upcoming calendar) — -// a zero count from it is expected, not a fetch failure. -const ORGS: Org[] = [ - { slug: 'mosaic', name: 'MOSAIC', host: 'mosaicbc.org', enabled: true }, - { slug: 'burnaby-nh', name: 'Burnaby Neighbourhood House', host: 'burnabynh.ca', enabled: true }, - { slug: 'success', name: 'S.U.C.C.E.S.S.', host: 'successbc.ca', enabled: true }, - { slug: 'centre-canada', name: 'CentreCanada', host: 'centrecanada.org', enabled: true }, - { slug: 'pirs', name: 'Pacific Immigrant Resources Society', host: 'pirs.bc.ca', enabled: true }, - // --- Phase 2, staged and NOT crawled until `enabled` flips (see Org.enabled) --- - // A public library rather than a settlement agency, so its calendar mixes genuinely - // relevant programming (English conversation, newcomer sessions) with toddler - // storytimes. A relevance filter for library sources lands with the Phase-2 adapter - // work; this org stays disabled until then, so nothing unfiltered can reach the tab. - { - slug: 'westvan-library', - name: 'West Vancouver Memorial Library', - host: 'westvanlibrary.ca', - enabled: false, - }, -]; - -/** - * The orgs a run actually crawls. Everything downstream — the fetch fan-out and the - * per-org counts in the response — is scoped to this, so a disabled org is inert rather - * than merely unreported. - */ -const ACTIVE_ORGS = ORGS.filter((org) => org.enabled); - -const MAX_PER_ORG = 25; // soonest-first cap so MOSAIC's ~254 events don't flood the tab -const MAX_DESCRIPTION_CHARS = 2000; -const MAX_TITLE_CHARS = 300; -const FETCH_TIMEOUT_MS = 20000; - -// Rolling window: only events starting between today and today + 4 months are ingested, -// so the tab neither fills with far-future placeholders nor grows without bound. Enforced -// twice — as Tribe's ?end_date (server side) and again per row in tribeEventToRow, since -// an org's API ignoring the param would otherwise slip rows past the bound. -// The web reader (services/community.ts) applies the same 4-month window on read; keep the -// two in step. Nothing is ever deleted: mobile reads this shared table with no date filter -// at all and has a Past tab, so aged-out rows must stay. -const WINDOW_MONTHS = 4; - -// How much of the description the genre tiebreaker scans. Long enough for the lede that -// actually describes the event, short enough to skip the registration/contact boilerplate. -const GENRE_DESCRIPTION_SCAN_CHARS = 400; - -// Tribe reads ?start_date on the SITE's calendar, not UTC — see todayInTimezone. -// Every Phase-1 org is in BC, so one constant covers them all. When an org outside -// Pacific time is added to ORGS, give Org an optional `timezone` and fall back here. -const ORG_TIMEZONE = 'America/Vancouver'; - -// Cover images are chosen in three tiers (see tribeEventToRow): -// 1. the event's own featured image (resolveImageUrl — icon-filtered, SSRF-guarded, -// size-checked), -// 2. a Pexels stock photo matching the event's topic (pexelsImage) — INERT unless the -// PEXELS_API_KEY edge-function secret is set, -// 3. this deterministic Unsplash pool, the always-available last resort. -// -// Topic-appropriate Unsplash fallback pool (images.unsplash.com is allowlisted in -// next.config.ts) for events whose source ships no usable cover. All ids are reused -// from the production news-crawler pools (already verified 200 image/jpeg) so none -// 404. Stable photo- form only. Per the never-null-image convention, but events -// keep the DB column nullable — the reused pool means we still supply one. -const IMG = (id: string) => - `https://images.unsplash.com/${id}?w=800&q=80&auto=format&fit=crop`; - -const EVENTS_FALLBACK_POOL: string[] = [ - 'photo-1517456793572-1d8efd6dc135', - 'photo-1697490251788-21888514f669', - 'photo-1517457373958-b7bdd4587205', - 'photo-1498661694102-0a3793edbe74', - 'photo-1642307063371-2e3e8909c3cb', - 'photo-1758272133542-b3107b947fc2', - 'photo-1523580846011-d3a5bc25702b', - 'photo-1531206715517-5c0ba140b2b8', - 'photo-1566438503908-4f8377461f58', - 'photo-1517048676732-d65bc937f952', - 'photo-1521791136064-7986c2920216', - 'photo-1573497491208-6b1acb260507', -].map(IMG); - -// Deterministic non-crypto string hash so a given event link always maps to the -// same pool image (stable across re-crawls) while different links spread across it. -function hashStr(s: string): number { - let h = 0; - for (let i = 0; i < s.length; i++) h = (h * 31 + s.charCodeAt(i)) | 0; - return Math.abs(h); -} - -function fallbackImage(link: string): string { - return EVENTS_FALLBACK_POOL[hashStr(link) % EVENTS_FALLBACK_POOL.length]; -} - -// Raw event titles ("Tai Chi – 48 Advanced", "QUEST+") are poor image-search terms, so map -// them to a topic query. Ordered: first matching keyword wins, generic settlement default -// otherwise. Keep the keywords lowercase — matched against the lowercased title. -const PEXELS_QUERY_RULES: Array<[RegExp, string]> = [ - [/english|conversation|language|esl/, 'english conversation class'], - [/job|career|employ|resume|hiring|worksafe|interview/, 'job fair career workshop'], - [/senior|55\+|elder|memory/, 'seniors community group'], - [/family|kids|child|parent|youth/, 'family community centre'], - [/tai chi|qi gong|dance|yoga|exercise|walk|fitness/, 'community exercise class'], - [/health|cancer|screening|wellness|clinic|mental/, 'community health workshop'], - [/housing|rental|tenant|co-op|home/, 'apartment housing keys'], - [/digital|computer|tech|online skill/, 'computer skills class'], - [/food|cook|cafe|meal|kitchen|dinner/, 'community kitchen cooking'], -]; -const PEXELS_QUERY_DEFAULT = 'community centre newcomers canada'; - -function pexelsQueryForEvent(title: string): string { - const t = title.toLowerCase(); - for (const [re, query] of PEXELS_QUERY_RULES) { - if (re.test(t)) return query; - } - return PEXELS_QUERY_DEFAULT; -} - -/** - * Tier-2 cover: a Pexels photo for the event's topic, or null if none (missing key / no - * results). `pexelsCache` memoises the candidate LIST per query for one crawler run, so the - * ~28 image-less events cost a handful of API calls; the per-event `seed` (hashStr of the - * link) then picks a stable photo from that list, keeping same-topic covers varied and - * unchanged across re-crawls. Caching the in-flight promise also dedupes concurrent calls. - */ -async function pexelsImage( - title: string, - seed: number, - cache: Map>, -): Promise { - const query = pexelsQueryForEvent(title); - let candidates = cache.get(query); - if (!candidates) { - candidates = fetchPexelsCandidates(query); - cache.set(query, candidates); - } - const urls = await candidates; - if (urls.length === 0) return null; - return urls[seed % urls.length]; -} - -// ---- genre tagging -------------------------------------------------------- - -// public.events.genre is a pre-existing shared column (default 'Uncategorized'). The -// mobile app types it as EventGenre and ships a genre chip filter that is currently -// commented out; the first five values below are exactly its enum, so nothing renames -// when Savar uncomments it — Language/Health/Family/Education are purely additive. -type EventGenre = - | 'Employment' - | 'Language' - | 'Housing' - | 'Finance' - | 'Documentation' - | 'Health' - | 'Family' - | 'Education' - | 'Socials' - | 'Uncategorized'; - -// Same shape as PEXELS_QUERY_RULES: ordered, first match wins, lowercase keywords. -// ORDER IS LOAD-BEARING: -// - Employment precedes Language because S.U.C.C.E.S.S. titles all carry -// "(English, Multilingual Translation Captions Available)" — matching Language -// first would file every employment workshop under Language. -// - Housing matches `housing`, never a bare `hous`, or "Belkin House" lands there. -// - Employment precedes Family so "Foreign Credential Recognition … Family Medicine -// Licensing" reads as a career event, not a family one. -// Verified against the 69 rows already crawled into prod: 69/69 classified, 0 fallthrough. -const GENRE_RULES: Array<[RegExp, EventGenre]> = [ - [ - /job|career|employ|resume|hiring|worksafe|workplace|interview|credential|licens|internationally (educated|trained)|profession|nurse|physician|labour market/, - 'Employment', - ], - [/english|\besl\b|\blinc\b|language|conversation circle|french|francais/, 'Language'], - [/housing|rental|renting|tenant|landlord|lease|shelter|homeless/, 'Housing'], - [ - /tax|bank|budget|financ|money|credit|benefit|insurance|pension|income|subsid|rrsp|tfsa|debt|saving/, - 'Finance', - ], - [ - /immigration|citizenship|permanent resident|pr card|work permit|study permit|sin card|legal|lawyer|notary|settlement|orientation|document|visa/, - 'Documentation', - ], - [ - /health|clinic|wellness|mental|counsel|cancer|screening|nutrition|dental|emotion|stress|mindful|yoga|tai chi|qi gong|exercise|fitness|walkathon|memory|dementia|therapy|doctor|medical|wellbeing/, - 'Health', - ], - [/famil|child|kid|parent|youth|toddler|baby|preschool|caregiver|prenatal|daycare/, 'Family'], - [/digital|computer|tech|literacy|skill|training|course|tutor|school|scholarship/, 'Education'], - // Last rule, so these keywords only ever catch events no earlier rule claimed. That's - // why the loose recreational-outing terms (explore, market, tour) are safe here and - // would not be higher up: "explore available pathways" and "labour market" already - // belong to Employment by the time this runs. - [ - /social|communit|cafe|café|club|dance|mahjong|party|celebrat|potluck|festival|connect|meetup|drop-?in|peer|volunteer|friend|game|craft|garden|coffee|lunch|dinner|cook|meal|kitchen|immigrant|refugee|newcomer|explore|tour\b|trip\b|outing|excursion|museum|farm|winery|orchard|hike|picnic|market|sightsee/, - 'Socials', - ], -]; - -function matchGenre(text: string): EventGenre | null { - for (const [re, genre] of GENRE_RULES) { - if (re.test(text)) return genre; - } - return null; -} - -/** - * Two passes: the title first, then the description as a tiebreaker. Title-only leaves - * opaque names ("QUEST+", "Seniors First BC", "Senior Farsi & Dari Program") uncategorized; - * matching both at once lets description boilerplate outvote a clear title signal. Running - * the description only when the title says nothing gets both — on the 69 rows already in - * prod just 5 fall through to the second pass. - */ -function genreForEvent(title: string, description: string | null): EventGenre { - return ( - matchGenre(title.toLowerCase()) ?? - matchGenre((description ?? '').slice(0, GENRE_DESCRIPTION_SCAN_CHARS).toLowerCase()) ?? - 'Uncategorized' - ); -} - -interface EventRow { - title: string; - description: string | null; - event_datetime: string; - event_end_datetime: string | null; - location: string; - event_type: 'in-person' | 'online' | 'hybrid'; - cover_photo_url: string | null; - external_link: string; - hosted_by: string | null; - address: string | null; - genre: EventGenre; - source: string; -} - -// ---- text helpers (dependency-free) --------------------------------------- - -/** - * `String.fromCodePoint` throws RangeError for anything outside 0…0x10FFFF — a feed - * carrying `�` or `�` is enough. That throw would escape - * decodeEntities() through clean()/htmlToParagraphs() and tribeEventToRow into - * fetchOrgEvents' catch, which returns [] — so ONE malformed entity in ONE item would - * discard that org's whole batch. Out-of-range values therefore keep their literal - * source text (`raw`) rather than throwing or being silently dropped. - */ -function codePointOr(raw: string, n: number): string { - return Number.isInteger(n) && n >= 0 && n <= 0x10ffff ? String.fromCodePoint(n) : raw; -} - -function decodeEntities(input: string): string { - return input - .replace(/&/g, '&') - .replace(/</g, '<') - .replace(/>/g, '>') - .replace(/"/g, '"') - .replace(/'/g, "'") - .replace(/�*39;/g, "'") - .replace(/�*27;/gi, "'") - .replace(/�*160;/g, ' ') - .replace(/ /g, ' ') - .replace(/–/g, '–') - .replace(/—/g, '—') - .replace(/‘|’/g, "'") - .replace(/“|”/g, '"') - .replace(/&#(\d+);/g, (m, n) => codePointOr(m, Number(n))) - .replace(/&#x([0-9a-f]+);/gi, (m, n) => codePointOr(m, parseInt(n, 16))); -} - -/** Decode entities, collapse whitespace, trim. For single-line fields (title, venue). */ -function clean(value: string | null | undefined): string { - if (!value) return ''; - return decodeEntities(String(value)).replace(/\s+/g, ' ').trim(); -} - -/** - * Convert an HTML fragment to plain text that PRESERVES paragraph breaks as `\n` - * (the event detail page splits description on `\n` to render paragraphs). Block-ish - * tags become newlines; remaining tags are stripped; entities decoded; runs of - * blank lines collapsed. Settlement-org bodies (esp. MOSAIC's Cvent form markup) - * carry heavy nesting, so we normalise aggressively. - */ -function htmlToParagraphs(html: string | null | undefined): string { - if (!html) return ''; - const withBreaks = String(html) - .replace(/<\s*br\s*\/?\s*>/gi, '\n') - .replace(/<\/\s*(p|div|li|ul|ol|h[1-6]|tr|blockquote)\s*>/gi, '\n') - .replace(/<[^>]*>/g, ' '); - return decodeEntities(withBreaks) - .split('\n') - .map((line) => line.replace(/\s+/g, ' ').trim()) - .filter(Boolean) - .join('\n') - .slice(0, MAX_DESCRIPTION_CHARS); -} - -// Venue names that denote a virtual room rather than a physical place. These orgs -// register a venue literally named "Online" or "Webinar" and never set Tribe's -// `is_virtual` (it needs the paid Virtual Events add-on), so the venue NAME is the only -// signal that an event is online — without this every webinar types as in-person. -const VIRTUAL_STRONG = new Set([ - 'online', 'virtual', 'webinar', 'zoom', 'remote', 'teleconference', 'livestream', - 'webex', 'teams', 'meet', -]); -const VIRTUAL_FILLER = new Set([ - 'event', 'events', 'meeting', 'session', 'workshop', 'only', 'platform', 'via', 'web', - 'google', 'ms', 'microsoft', 'stream', 'live', -]); - -/** - * True when the whole venue name is virtual vocabulary ("Online", "Webinar", - * "Online (Zoom)", "Virtual Event"). Matches the ENTIRE name rather than searching for a - * substring, so a physical venue that merely contains one of these words — "Online - * Learning Centre, 123 Main St" — is not misread as virtual. - */ -function isOnlineVenueName(name: string): boolean { - const tokens = name - .toLowerCase() - .replace(/[^a-z0-9]+/g, ' ') - .trim() - .split(/\s+/) - .filter(Boolean); - if (tokens.length === 0) return false; - if (!tokens.every((t) => VIRTUAL_STRONG.has(t) || VIRTUAL_FILLER.has(t))) return false; - return tokens.some((t) => VIRTUAL_STRONG.has(t)); -} - -/** Tribe `utc_start_date`/`utc_end_date` ("YYYY-MM-DD HH:MM:SS", UTC) → ISO Z. */ -function toIsoUtc(value: string | null | undefined): string | null { - if (!value || typeof value !== 'string') return null; - const m = value.trim().match(/^(\d{4}-\d{2}-\d{2})[ T](\d{2}:\d{2}:\d{2})/); - if (!m) return null; - const iso = `${m[1]}T${m[2]}Z`; - const d = new Date(iso); - return Number.isNaN(d.getTime()) ? null : d.toISOString(); -} - -// ---- image quality trio (reused from news-crawler) ------------------------ - -const ICON_URL_RE = - /logo|icon|brand|favicon|placeholder|avatar|trademark|\/tm(?:[\/?#._-]|$)|spinner|badge|wp-content\/uploads.*logo/i; -const MIN_IMAGE_BYTES = 10 * 1024; -const IMAGE_HEAD_TIMEOUT_MS = 3000; - -/** - * Keep a source image only if it looks like a real photo: reject icon-ish URLs, - * SSRF-guard, then HEAD-check the byte size (3s cap). Reject only on a positive - * Content-Length below the threshold — a missing header, non-OK status, or a - * thrown/timed-out request all keep the image (benefit of the doubt). Returns null - * when there's no usable photo, so the caller applies the fallback pool. - * - * `redirect: 'manual'` completes the SSRF guard: without it an allowed public host - * could 302 the probe to an internal address, which isPublicHttpUrl only ever saw - * the ORIGINAL URL for. A 3xx now surfaces as a non-OK response, so we keep the - * original (already-validated) URL and never fetch or store the redirect target. - * Cost: a redirected image skips the size check — the same benefit-of-the-doubt - * the non-OK path already takes, and cheaper than dropping the many legitimate - * http→https / CDN redirects these WordPress hosts serve. - */ -async function resolveImageUrl(url: string | null): Promise { - if (!url || typeof url !== 'string') return null; - if (ICON_URL_RE.test(url)) return null; - if (!isPublicHttpUrl(url)) return null; // SSRF guard before the HEAD fetch - - const controller = new AbortController(); - const timer = setTimeout(() => controller.abort(), IMAGE_HEAD_TIMEOUT_MS); - try { - const res = await fetch(url, { - method: 'HEAD', - signal: controller.signal, - redirect: 'manual', // never follow a redirect off the validated host - headers: { 'User-Agent': 'UnifyEventsBot/1.0 (+https://unifysocial.ca)' }, - }); - if (!res.ok) return url; // non-OK ⇒ keep (benefit of the doubt) - const lenHeader = res.headers.get('content-length'); - if (lenHeader !== null) { - const bytes = Number(lenHeader); - if (Number.isFinite(bytes) && bytes < MIN_IMAGE_BYTES) return null; - } - return url; - } catch { - return url; - } finally { - clearTimeout(timer); - } -} - -// ---- Tribe REST → EventRow ------------------------------------------------- - -function organizerName(organizer: unknown): string | null { - if (!Array.isArray(organizer) || organizer.length === 0) return null; - const first = organizer[0]; - if (typeof first === 'string') return clean(first) || null; - if (first && typeof first === 'object' && typeof (first as any).organizer === 'string') { - return clean((first as any).organizer) || null; - } - return null; -} - -/** - * Map one Tribe REST event object to an EventRow, or null when it can't be shaped - * into a valid row (missing NOT-NULL data, undeterminable location, past, hidden, - * or starting beyond the rolling window). - * Image resolution is async (HEAD check + optional Pexels search), so this is async. - * `pexelsCache` is the run-scoped per-query memo threaded down from the handler. - */ -async function tribeEventToRow( - ev: any, - org: Org, - pexelsCache: Map>, - windowEndMs: number, -): Promise { - if (!ev || typeof ev !== 'object') return null; - if (ev.status && ev.status !== 'publish') return null; - if (ev.hide_from_listings === true) return null; - - const title = clean(ev.title).slice(0, MAX_TITLE_CHARS); - const externalLink = typeof ev.url === 'string' ? ev.url.trim() : ''; - const eventDatetime = toIsoUtc(ev.utc_start_date); - if (!title || !externalLink || !eventDatetime) return null; // NOT NULL columns - - // Backstop for ?end_date — see fetchOrgEvents. Runs before the image work below so a - // far-future event costs no HEAD probe or Pexels lookup. - if (Date.parse(eventDatetime) > windowEndMs) return null; - - // location + event_type from the venue NAME first, then venue presence, then the - // virtual flag — see isOnlineVenueName for why the name has to win. - const venue = ev.venue; - const hasVenue = - venue && typeof venue === 'object' && !Array.isArray(venue) && !!clean(venue.venue); - const isVirtual = ev.is_virtual === true; - const venueName = hasVenue ? clean(venue.venue) : ''; - - let location: string; - let eventType: EventRow['event_type']; - if (hasVenue && isOnlineVenueName(venueName)) { - location = venueName; // keep the source wording ('Online' / 'Webinar') - eventType = 'online'; - } else if (hasVenue) { - location = venueName; - eventType = isVirtual ? 'hybrid' : 'in-person'; - } else if (isVirtual) { - location = 'Online'; - eventType = 'online'; - } else { - return null; // neither a venue nor a virtual flag → no reliable location - } - - // description: prefer the (usually cleaner) excerpt, fall back to the body. - const excerpt = htmlToParagraphs(ev.excerpt); - const body = htmlToParagraphs(ev.description); - const description = (excerpt.length >= 40 ? excerpt : body) || null; - - const address = - hasVenue && clean(venue.address) - ? [venue.address, venue.city, venue.stateprovince || venue.province, venue.zip] - .map((p: unknown) => clean(p as string)) - .filter(Boolean) - .join(', ') - : null; - - const imageUrl = - ev.image && typeof ev.image === 'object' && typeof ev.image.url === 'string' - ? ev.image.url - : null; - // Three tiers: source image → Pexels topic photo → deterministic Unsplash pool. - const seed = hashStr(externalLink); - const coverPhotoUrl = - (await resolveImageUrl(imageUrl)) ?? - (await pexelsImage(title, seed, pexelsCache)) ?? - fallbackImage(externalLink); - - return { - title, - description, - event_datetime: eventDatetime, - event_end_datetime: toIsoUtc(ev.utc_end_date), - location, - event_type: eventType, - cover_photo_url: coverPhotoUrl, - external_link: externalLink, - hosted_by: organizerName(ev.organizer) ?? org.name, - address, - genre: genreForEvent(title, description), - source: `crawler:${org.slug}`, - }; -} - -/** - * Today's date as YYYY-MM-DD on `timeZone`'s calendar, NOT UTC's. - * - * Tribe interprets ?start_date in the site's own timezone, so using the UTC date - * would silently drop events happening later the same local day for any run between - * 00:00Z and 08:00Z — the window where UTC has already rolled over to tomorrow but - * Vancouver has not. The scheduled Monday 14:00 UTC run is outside that window, but a - * manual or rescheduled run lands in it easily. 'en-CA' already formats as YYYY-MM-DD, - * so no manual part assembly is needed. - */ -function todayInTimezone(timeZone: string, now: Date = new Date()): string { - return new Intl.DateTimeFormat('en-CA', { - timeZone, - year: 'numeric', - month: '2-digit', - day: '2-digit', - }).format(now); -} - -/** - * The far edge of the rolling window as YYYY-MM-DD, `months` after today on `timeZone`'s - * calendar — the ?end_date counterpart to todayInTimezone's ?start_date. - * - * Date.UTC normalises overflow, so a short target month rolls forward rather than throwing - * (Oct 31 + 4 months → Feb 31 → Mar 3). A couple of days of slack on the far edge of a - * four-month bound doesn't matter; being off by a whole month would. - */ -function windowEndInTimezone( - timeZone: string, - months: number, - now: Date = new Date(), -): string { - const [year, month, day] = todayInTimezone(timeZone, now).split('-').map(Number); - return new Date(Date.UTC(year, month - 1 + months, day)).toISOString().slice(0, 10); -} - -async function fetchOrgEvents( - org: Org, - pexelsCache: Map>, -): Promise { - const today = todayInTimezone(ORG_TIMEZONE); // YYYY-MM-DD on the org's calendar - const windowEnd = windowEndInTimezone(ORG_TIMEZONE, WINDOW_MONTHS); - // Deliberately loose: parsed as UTC while the date itself is on the org's calendar, so - // the per-row guard trails ?end_date by a few hours. It exists to catch an org whose API - // ignores end_date outright (events years out), not to police the boundary hour. - const windowEndMs = Date.parse(`${windowEnd}T23:59:59Z`); - const url = - `https://${org.host}/wp-json/tribe/events/v1/events` + - `?per_page=${MAX_PER_ORG}&start_date=${today}%2000:00:00&end_date=${windowEnd}%2023:59:59`; - - const controller = new AbortController(); - const timer = setTimeout(() => controller.abort(), FETCH_TIMEOUT_MS); - try { - const res = await fetch(url, { - signal: controller.signal, - headers: { - 'User-Agent': 'UnifyEventsBot/1.0 (+https://unifysocial.ca)', - Accept: 'application/json', - }, - }); - if (!res.ok) { - console.error(`events-crawler: ${org.slug} returned HTTP ${res.status}`); - return []; - } - const data = await res.json(); - const events = Array.isArray(data?.events) ? data.events : []; - const rows = await Promise.all( - events - .slice(0, MAX_PER_ORG) - .map((ev: unknown) => tribeEventToRow(ev, org, pexelsCache, windowEndMs)), - ); - return rows.filter((r): r is EventRow => r !== null); - } catch (error) { - console.error(`events-crawler: failed to fetch ${org.slug}:`, error); - return []; - } finally { - clearTimeout(timer); - } -} - function jsonResponse(body: Record, status = 200): Response { return new Response(JSON.stringify(body), { status, @@ -636,10 +35,9 @@ function jsonResponse(body: Record, status = 200): Response { } /** - * Read a JWT payload's `role` claim WITHOUT verifying the signature. Safe to trust - * ONLY because this function runs with verify_jwt=true (config.toml) — the gateway - * has already verified the signature before we get here. Returns null on any - * malformed token. + * Read a JWT payload's `role` claim WITHOUT verifying the signature. Safe to trust ONLY + * because this function runs with verify_jwt=true (config.toml) — the gateway has already + * verified the signature before we get here. Returns null on any malformed token. */ function jwtRole(token: string): string | null { try { @@ -654,8 +52,6 @@ function jwtRole(token: string): string | null { } } -// ---- handler -------------------------------------------------------------- - Deno.serve(async (req: Request) => { const supabaseUrl = Deno.env.get('SUPABASE_URL'); const serviceRoleKey = Deno.env.get('SUPABASE_SERVICE_ROLE_KEY'); @@ -669,8 +65,7 @@ Deno.serve(async (req: Request) => { // rejection, log the presented role (never the token). const token = req.headers.get('Authorization')?.replace('Bearer ', '').trim(); const authorized = - !!token && - (token === serviceRoleKey.trim() || jwtRole(token) === 'service_role'); + !!token && (token === serviceRoleKey.trim() || jwtRole(token) === 'service_role'); if (!authorized) { console.error( `events-crawler: unauthorized — bearer role=${ @@ -682,19 +77,18 @@ Deno.serve(async (req: Request) => { const supabase = createClient(supabaseUrl, serviceRoleKey); - // Fetch every org concurrently. One Pexels candidate-list cache per run, shared across - // all orgs, so identical topic queries hit the API once. - const pexelsCache = new Map>(); - const perOrgResults = await Promise.all( - ACTIVE_ORGS.map(async (org) => ({ - slug: org.slug, - rows: await fetchOrgEvents(org, pexelsCache), + const ctx = makeContext(); + + const perSourceResults = await Promise.all( + ACTIVE_SOURCES.map(async (source) => ({ + slug: source.slug, + rows: await ADAPTERS[source.kind](source, ctx), })), ); const collected: EventRow[] = []; const perOrg: Record = {}; - for (const { slug, rows } of perOrgResults) { + for (const { slug, rows } of perSourceResults) { perOrg[slug] = rows.length; collected.push(...rows); } @@ -712,20 +106,20 @@ Deno.serve(async (req: Request) => { } // Dedupe against the DB by external_link. The real guarantee is the unique index - // events_external_link_key (20260724130000_events_external_link_unique.sql) paired - // with the ignoreDuplicates upsert below — a read-then-insert alone would race, since - // a manual trigger overlapping the cron could have both runs pass this check. This - // filter is defence-in-depth: it keeps the common case from shipping rows the DB - // would only discard, and it keeps `inserted` meaningful. + // events_external_link_key (20260724130000_events_external_link_unique.sql) paired with + // the ignoreDuplicates upsert below — a read-then-insert alone would race, since a + // manual trigger overlapping the cron could have both runs pass this check. This filter + // is defence-in-depth: it keeps the common case from shipping rows the DB would only + // discard, and it keeps `inserted` meaningful. // - // Scoped to THIS batch's links rather than selecting the whole column, so the lookup - // can never be silently truncated by the API row cap as the table grows. Scoping by - // link (not by source) keeps the manual-row protection: a link Savar entered by hand - // still blocks a crawler re-insert. + // Scoped to THIS batch's links rather than selecting the whole column, so the lookup can + // never be silently truncated by the API row cap as the table grows. Scoping by link + // (not by source) keeps the manual-row protection: a link Savar entered by hand still + // blocks a crawler re-insert. // - // Chunked because the filter travels in a GET query string: links run ~140 chars - // (~200 URL-encoded), so 25 per request keeps it near 5KB, well under the gateway's - // ~8KB request-line cap. Batch max is ORGS.length * MAX_PER_ORG = 125 → ≤5 calls. + // Chunked because the filter travels in a GET query string: links run ~140 chars (~200 + // URL-encoded), so 25 per request keeps it near 5KB, well under the gateway's ~8KB + // request-line cap. const EXISTING_LOOKUP_CHUNK = 25; const existingLinks = new Set(); for (let i = 0; i < batch.length; i += EXISTING_LOOKUP_CHUNK) { @@ -748,8 +142,8 @@ Deno.serve(async (req: Request) => { return jsonResponse({ ok: true, perOrg, fetched: batch.length, inserted: 0 }); } - // ON CONFLICT (external_link) DO NOTHING via the UNIQUE index, so a run that loses - // the race to a concurrent invocation is a silent no-op instead of a 23505. .select() + // ON CONFLICT (external_link) DO NOTHING via the UNIQUE index, so a run that loses the + // race to a concurrent invocation is a silent no-op instead of a 23505. .select() // returns only the rows actually inserted, so its length is the true insert count. const { data, error } = await supabase .from('events') diff --git a/supabase/functions/events-crawler/lib/constants.ts b/supabase/functions/events-crawler/lib/constants.ts new file mode 100644 index 0000000..bb015cf --- /dev/null +++ b/supabase/functions/events-crawler/lib/constants.ts @@ -0,0 +1,39 @@ +// Tunables shared by the crawler handler and every adapter. Kept in one module so a +// limit can't drift between two sources that are meant to behave the same way. + +/** Soonest-first cap per source, so one busy calendar can't flood the Events tab. */ +export const MAX_PER_ORG = 25; + +export const MAX_DESCRIPTION_CHARS = 2000; +export const MAX_TITLE_CHARS = 300; +export const FETCH_TIMEOUT_MS = 20000; + +/** + * Rolling window: only events starting between today and today + WINDOW_MONTHS are + * ingested, so the tab neither fills with far-future placeholders nor grows without + * bound. Enforced twice — as a server-side query bound where the source supports one, + * and again per row in every adapter, since a source that ignores the bound (or has no + * way to express it, like BiblioCommons) would otherwise slip rows past it. + * + * The web reader (services/community.ts, EVENTS_WINDOW_MONTHS) applies the same window + * on read; keep the two in step. Nothing is ever deleted: mobile reads this shared table + * with no date filter at all and has a Past tab, so aged-out rows must stay. + */ +export const WINDOW_MONTHS = 4; + +/** + * How much of the description the genre tiebreaker scans. Long enough for the lede that + * actually describes the event, short enough to skip registration/contact boilerplate. + */ +export const GENRE_DESCRIPTION_SCAN_CHARS = 400; + +/** + * Sources interpret a date-bound query on their own calendar, not UTC — see + * todayInTimezone. Every source so far is in BC, so one constant covers them all. A + * source outside Pacific time should carry its own `timezone` on the Source record and + * fall back to this. + */ +export const ORG_TIMEZONE = 'America/Vancouver'; + +/** Sent on every outbound request so source operators can identify (and contact) us. */ +export const USER_AGENT = 'UnifyEventsBot/1.0 (+https://unifysocial.ca)'; diff --git a/supabase/functions/events-crawler/lib/dates.ts b/supabase/functions/events-crawler/lib/dates.ts new file mode 100644 index 0000000..2eb7c28 --- /dev/null +++ b/supabase/functions/events-crawler/lib/dates.ts @@ -0,0 +1,61 @@ +// Date helpers. Everything stored in public.events is UTC ISO; everything used to bound +// a source query is a calendar date in the source's own timezone. + +/** "YYYY-MM-DD HH:MM:SS" or ISO-ish (UTC) → ISO Z. Returns null on anything unparseable. */ +export function toIsoUtc(value: string | null | undefined): string | null { + if (!value || typeof value !== 'string') return null; + const m = value.trim().match(/^(\d{4}-\d{2}-\d{2})[ T](\d{2}:\d{2}:\d{2})/); + if (!m) return null; + const iso = `${m[1]}T${m[2]}Z`; + const d = new Date(iso); + return Number.isNaN(d.getTime()) ? null : d.toISOString(); +} + +/** + * An ISO-8601 timestamp that already carries its own offset ("2026-08-20T18:15:00Z", + * "2026-07-30T09:00:00-07:00") → ISO Z. Distinct from toIsoUtc, which assumes UTC for + * offset-less input; feeding an offset-bearing string to that one would silently shift + * the time by the offset. + */ +export function offsetIsoToUtc(value: string | null | undefined): string | null { + if (!value || typeof value !== 'string') return null; + const trimmed = value.trim(); + if (!/(?:Z|[+-]\d{2}:?\d{2})$/i.test(trimmed)) return null; + const d = new Date(trimmed); + return Number.isNaN(d.getTime()) ? null : d.toISOString(); +} + +/** + * Today's date as YYYY-MM-DD on `timeZone`'s calendar, NOT UTC's. + * + * Sources interpret a date bound on their own calendar, so using the UTC date would + * silently drop events happening later the same local day for any run between 00:00Z and + * 08:00Z — the window where UTC has already rolled over to tomorrow but Vancouver has + * not. The scheduled Monday 14:00 UTC run is outside that window, but a manual or + * rescheduled run lands in it easily. 'en-CA' already formats as YYYY-MM-DD. + */ +export function todayInTimezone(timeZone: string, now: Date = new Date()): string { + return new Intl.DateTimeFormat('en-CA', { + timeZone, + year: 'numeric', + month: '2-digit', + day: '2-digit', + }).format(now); +} + +/** + * The far edge of the rolling window as YYYY-MM-DD, `months` after today on `timeZone`'s + * calendar — the counterpart to todayInTimezone. + * + * Date.UTC normalises overflow, so a short target month rolls forward rather than + * throwing (Oct 31 + 4 months → Feb 31 → Mar 3). A couple of days of slack on the far + * edge of a four-month bound doesn't matter; being off by a whole month would. + */ +export function windowEndInTimezone( + timeZone: string, + months: number, + now: Date = new Date(), +): string { + const [year, month, day] = todayInTimezone(timeZone, now).split('-').map(Number); + return new Date(Date.UTC(year, month - 1 + months, day)).toISOString().slice(0, 10); +} diff --git a/supabase/functions/events-crawler/lib/genre.ts b/supabase/functions/events-crawler/lib/genre.ts new file mode 100644 index 0000000..6961e70 --- /dev/null +++ b/supabase/functions/events-crawler/lib/genre.ts @@ -0,0 +1,66 @@ +// Genre tagging. public.events.genre is a pre-existing shared column that both apps read. + +import { GENRE_DESCRIPTION_SCAN_CHARS } from './constants.ts'; +import type { EventGenre } from './types.ts'; + +// Ordered, first match wins, lowercase keywords. +// ORDER IS LOAD-BEARING: +// - Employment precedes Language because S.U.C.C.E.S.S. titles all carry +// "(English, Multilingual Translation Captions Available)" — matching Language +// first would file every employment workshop under Language. +// - Housing matches `housing`, never a bare `hous`, or "Belkin House" lands there. +// - Employment precedes Family so "Foreign Credential Recognition … Family Medicine +// Licensing" reads as a career event, not a family one. +// Verified against the 69 rows already crawled into prod: 69/69 classified, 0 fallthrough. +const GENRE_RULES: Array<[RegExp, EventGenre]> = [ + [ + /job|career|employ|resume|hiring|worksafe|workplace|interview|credential|licens|internationally (educated|trained)|profession|nurse|physician|labour market/, + 'Employment', + ], + [/english|\besl\b|\blinc\b|language|conversation circle|french|francais/, 'Language'], + [/housing|rental|renting|tenant|landlord|lease|shelter|homeless/, 'Housing'], + [ + /tax|bank|budget|financ|money|credit|benefit|insurance|pension|income|subsid|rrsp|tfsa|debt|saving/, + 'Finance', + ], + [ + /immigration|citizenship|permanent resident|pr card|work permit|study permit|sin card|legal|lawyer|notary|settlement|orientation|document|visa/, + 'Documentation', + ], + [ + /health|clinic|wellness|mental|counsel|cancer|screening|nutrition|dental|emotion|stress|mindful|yoga|tai chi|qi gong|exercise|fitness|walkathon|memory|dementia|therapy|doctor|medical|wellbeing/, + 'Health', + ], + [/famil|child|kid|parent|youth|toddler|baby|preschool|caregiver|prenatal|daycare/, 'Family'], + [/digital|computer|tech|literacy|skill|training|course|tutor|school|scholarship/, 'Education'], + // Last rule, so these keywords only ever catch events no earlier rule claimed. That's + // why the loose recreational-outing terms (explore, market, tour) are safe here and + // would not be higher up: "explore available pathways" and "labour market" already + // belong to Employment by the time this runs. + [ + /social|communit|cafe|café|club|dance|mahjong|party|celebrat|potluck|festival|connect|meetup|drop-?in|peer|volunteer|friend|game|craft|garden|coffee|lunch|dinner|cook|meal|kitchen|immigrant|refugee|newcomer|explore|tour\b|trip\b|outing|excursion|museum|farm|winery|orchard|hike|picnic|market|sightsee/, + 'Socials', + ], +]; + +function matchGenre(text: string): EventGenre | null { + for (const [re, genre] of GENRE_RULES) { + if (re.test(text)) return genre; + } + return null; +} + +/** + * Two passes: the title first, then the description as a tiebreaker. Title-only leaves + * opaque names ("QUEST+", "Seniors First BC", "Senior Farsi & Dari Program") + * uncategorized; matching both at once lets description boilerplate outvote a clear title + * signal. Running the description only when the title says nothing gets both — on the 69 + * rows already in prod just 5 fall through to the second pass. + */ +export function genreForEvent(title: string, description: string | null): EventGenre { + return ( + matchGenre(title.toLowerCase()) ?? + matchGenre((description ?? '').slice(0, GENRE_DESCRIPTION_SCAN_CHARS).toLowerCase()) ?? + 'Uncategorized' + ); +} diff --git a/supabase/functions/events-crawler/lib/images.ts b/supabase/functions/events-crawler/lib/images.ts new file mode 100644 index 0000000..9f40aa5 --- /dev/null +++ b/supabase/functions/events-crawler/lib/images.ts @@ -0,0 +1,156 @@ +// Cover-image resolution, in three tiers: +// 1. the event's own image (resolveImageUrl — icon-filtered, SSRF-guarded, size-checked), +// 2. a Pexels stock photo matching the event's topic — INERT unless PEXELS_API_KEY is set, +// 3. a deterministic Unsplash pool, the always-available last resort. + +import { isPublicHttpUrl } from '../../_shared/ssrf.ts'; +import { fetchPexelsCandidates } from '../../_shared/pexels.ts'; +import { USER_AGENT } from './constants.ts'; + +// Topic-appropriate Unsplash fallback pool (images.unsplash.com is allowlisted in +// next.config.ts). All ids are reused from the production news-crawler pools (already +// verified 200 image/jpeg) so none 404. Stable photo- form only. +const IMG = (id: string) => + `https://images.unsplash.com/${id}?w=800&q=80&auto=format&fit=crop`; + +const EVENTS_FALLBACK_POOL: string[] = [ + 'photo-1517456793572-1d8efd6dc135', + 'photo-1697490251788-21888514f669', + 'photo-1517457373958-b7bdd4587205', + 'photo-1498661694102-0a3793edbe74', + 'photo-1642307063371-2e3e8909c3cb', + 'photo-1758272133542-b3107b947fc2', + 'photo-1523580846011-d3a5bc25702b', + 'photo-1531206715517-5c0ba140b2b8', + 'photo-1566438503908-4f8377461f58', + 'photo-1517048676732-d65bc937f952', + 'photo-1521791136064-7986c2920216', + 'photo-1573497491208-6b1acb260507', +].map(IMG); + +/** + * Deterministic non-crypto string hash so a given event link always maps to the same + * pool image (stable across re-crawls) while different links spread across it. + */ +export function hashStr(s: string): number { + let h = 0; + for (let i = 0; i < s.length; i++) h = (h * 31 + s.charCodeAt(i)) | 0; + return Math.abs(h); +} + +export function fallbackImage(link: string): string { + return EVENTS_FALLBACK_POOL[hashStr(link) % EVENTS_FALLBACK_POOL.length]; +} + +// Raw event titles ("Tai Chi – 48 Advanced", "QUEST+") are poor image-search terms, so +// map them to a topic query. Ordered: first matching keyword wins, generic settlement +// default otherwise. Keep the keywords lowercase — matched against the lowercased title. +const PEXELS_QUERY_RULES: Array<[RegExp, string]> = [ + [/english|conversation|language|esl/, 'english conversation class'], + [/job|career|employ|resume|hiring|worksafe|interview/, 'job fair career workshop'], + [/senior|55\+|elder|memory/, 'seniors community group'], + [/family|kids|child|parent|youth/, 'family community centre'], + [/tai chi|qi gong|dance|yoga|exercise|walk|fitness/, 'community exercise class'], + [/health|cancer|screening|wellness|clinic|mental/, 'community health workshop'], + [/housing|rental|tenant|co-op|home/, 'apartment housing keys'], + [/digital|computer|tech|online skill/, 'computer skills class'], + [/food|cook|cafe|meal|kitchen|dinner/, 'community kitchen cooking'], +]; +const PEXELS_QUERY_DEFAULT = 'community centre newcomers canada'; + +function pexelsQueryForEvent(title: string): string { + const t = title.toLowerCase(); + for (const [re, query] of PEXELS_QUERY_RULES) { + if (re.test(t)) return query; + } + return PEXELS_QUERY_DEFAULT; +} + +/** + * Tier-2 cover: a Pexels photo for the event's topic, or null if none (missing key / no + * results). `cache` memoises the candidate LIST per query for one crawler run, so the + * image-less events cost a handful of API calls; the per-event `seed` (hashStr of the + * link) then picks a stable photo from that list, keeping same-topic covers varied and + * unchanged across re-crawls. Caching the in-flight promise also dedupes concurrent calls. + */ +export async function pexelsImage( + title: string, + seed: number, + cache: Map>, +): Promise { + const query = pexelsQueryForEvent(title); + let candidates = cache.get(query); + if (!candidates) { + candidates = fetchPexelsCandidates(query); + cache.set(query, candidates); + } + const urls = await candidates; + if (urls.length === 0) return null; + return urls[seed % urls.length]; +} + +const ICON_URL_RE = + /logo|icon|brand|favicon|placeholder|avatar|trademark|\/tm(?:[\/?#._-]|$)|spinner|badge|wp-content\/uploads.*logo/i; +const MIN_IMAGE_BYTES = 10 * 1024; +const IMAGE_HEAD_TIMEOUT_MS = 3000; + +/** + * Keep a source image only if it looks like a real photo: reject icon-ish URLs, + * SSRF-guard, then HEAD-check the byte size (3s cap). Reject only on a positive + * Content-Length below the threshold — a missing header, non-OK status, or a + * thrown/timed-out request all keep the image (benefit of the doubt). Returns null when + * there's no usable photo, so the caller applies the next tier. + * + * `redirect: 'manual'` completes the SSRF guard: without it an allowed public host could + * 302 the probe to an internal address, which isPublicHttpUrl only ever saw the ORIGINAL + * URL for. A 3xx now surfaces as a non-OK response, so we keep the original + * (already-validated) URL and never fetch or store the redirect target. Cost: a + * redirected image skips the size check — the same benefit-of-the-doubt the non-OK path + * already takes, and cheaper than dropping the many legitimate http→https / CDN + * redirects these hosts serve. + */ +export async function resolveImageUrl(url: string | null): Promise { + if (!url || typeof url !== 'string') return null; + if (ICON_URL_RE.test(url)) return null; + if (!isPublicHttpUrl(url)) return null; // SSRF guard before the HEAD fetch + + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), IMAGE_HEAD_TIMEOUT_MS); + try { + const res = await fetch(url, { + method: 'HEAD', + signal: controller.signal, + redirect: 'manual', // never follow a redirect off the validated host + headers: { 'User-Agent': USER_AGENT }, + }); + if (!res.ok) return url; // non-OK ⇒ keep (benefit of the doubt) + const lenHeader = res.headers.get('content-length'); + if (lenHeader !== null) { + const bytes = Number(lenHeader); + if (Number.isFinite(bytes) && bytes < MIN_IMAGE_BYTES) return null; + } + return url; + } catch { + return url; + } finally { + clearTimeout(timer); + } +} + +/** + * The full three-tier resolution every adapter uses. `sourceImage` may be null when the + * feed ships none — the Pexels and Unsplash tiers still guarantee a cover. + */ +export async function resolveCover( + sourceImage: string | null, + title: string, + externalLink: string, + cache: Map>, +): Promise { + const seed = hashStr(externalLink); + return ( + (await resolveImageUrl(sourceImage)) ?? + (await pexelsImage(title, seed, cache)) ?? + fallbackImage(externalLink) + ); +} diff --git a/supabase/functions/events-crawler/lib/relevance.ts b/supabase/functions/events-crawler/lib/relevance.ts new file mode 100644 index 0000000..5377404 --- /dev/null +++ b/supabase/functions/events-crawler/lib/relevance.ts @@ -0,0 +1,75 @@ +// Settlement-relevance filter for library and university sources. +// +// WHY: the five Phase-1 sources are settlement agencies — everything they publish is +// on-mission, so they are never filtered. Libraries and universities publish a general +// programming calendar in which settlement content is a small minority: measured against +// the live feeds, storytimes, baby/toddler programs, teen gaming, board games, gallery +// talks and thesis defences make up the large majority. Ingesting those unfiltered would +// turn the Events tab into a library what's-on listing, burying the settlement events +// this app exists to surface. +// +// TITLE ONLY, deliberately. Matching descriptions too was measured and rejected: library +// blurbs routinely close with lines like "newcomer families welcome", which pulls in +// every storytime and defeats the filter. The cost is real but small — an opaquely-named +// but genuinely relevant event (NVDPL's "Open Door Community Hub Drop-In") is dropped. +// Precision is the right trade here, because a missed event is invisible while a flood of +// irrelevant ones is actively harmful. Revisit by adding terms, not by widening to +// descriptions. +// +// Measured keep-rates on the live feeds when this landed (2026-07-31): +// West Van 8/50 · VPL 6/100 · SFU 5/200 · NVDPL 16/100 · Surrey 2/10 (page 1) + +/** + * Accents are stripped before matching (NFD, then drop combining marks), so a single + * ASCII pattern covers every accented spelling a source might use. This is load-bearing: + * Surrey publishes "WorkBC Résumé Clinic" and NVDPL publishes "Tech Café", neither of + * which would match a plain `resume` / `cafe` term without it. + */ +function foldAccents(value: string): string { + // \u0300-\u036f is the Combining Diacritical Marks block, written escaped because the + // literal characters are invisible in an editor and trivially corrupted by a bad paste. + return value.normalize('NFD').replace(/[\u0300-\u036f]/g, ''); +} + +/** + * Grouped for reviewability — the groups are purely documentation, the union is what + * runs. Terms are matched case-insensitively against the accent-folded title. + */ +const RELEVANCE_RE = new RegExp( + [ + // Status & settlement + 'newcomer|immigrant|immigration|refugee|asylum|settlement|migrant|international\\s+student', + // Legal & documents + 'citizenship|permanent\\s+resident|pr\\s+card|work\\s+permit|study\\s+permit|visa\\b', + 'social\\s+insurance|sin\\s+clinic|legal\\s+clinic|notary', + // Language + '\\besl\\b|\\blinc\\b|english\\s+(conversation|class|corner|practice|language)', + 'conversation\\s+(circle|club|cafe)|language\\s+(exchange|cafe)|speaking\\s+english|literacy', + // Employment + 'job\\s+(search|fair|club|help)|resume|cover\\s+letter|interview\\s+skills|career', + 'employment|workbc|hiring|credential|foreign[-\\s]trained|workplace', + // Finance + 'tax\\s+clinic|income\\s+tax|banking|bank\\s+account|budgeting|financial\\s+literacy', + 'credit\\s+score', + // Housing + 'housing|tenant|tenancy|rental|renting|landlord|lease\\b', + // Digital literacy + 'digital\\s+literacy|computer\\s+(basics|skills|help)|tech\\s+(cafe|help|support)', + 'device\\s+clinic|online\\s+safety|internet\\s+basics', + // Health-system navigation + 'health\\s+card|family\\s+doctor|\\bmsp\\b|medical\\s+insurance', + ].join('|'), + 'i', +); + +/** + * Whether a library/university event's title indicates settlement-relevant content. + * + * Note what is deliberately absent: a bare `orientation` would catch West Van's + * "Recording Studio Orientation", so SFU's flagship "International Student Orientation" + * is caught by `international student` instead. Likewise bare `health`, `family` and + * `community` are excluded as far too broad for a library calendar. + */ +export function isSettlementRelevant(title: string): boolean { + return RELEVANCE_RE.test(foldAccents(title)); +} diff --git a/supabase/functions/events-crawler/lib/sources.ts b/supabase/functions/events-crawler/lib/sources.ts new file mode 100644 index 0000000..1762aeb --- /dev/null +++ b/supabase/functions/events-crawler/lib/sources.ts @@ -0,0 +1,107 @@ +// The source registry, the adapter map, and the per-run context factory. +// +// Side-effect free on purpose: index.ts calls Deno.serve at module scope, so anything that +// needs this wiring without starting a server (the dryrun harness) has to import it from +// here. Keeping one definition means the preview and a real run cannot drift. + +import * as bibliocommons from '../adapters/bibliocommons.ts'; +import * as tribe from '../adapters/tribe.ts'; +import { ORG_TIMEZONE, WINDOW_MONTHS } from './constants.ts'; +import { todayInTimezone, windowEndInTimezone } from './dates.ts'; +import type { Adapter, AdapterContext, Source, SourceKind } from './types.ts'; + +// Phase 1 (enabled): the five highest-relevance orgs, all on The Events Calendar (Tribe). +// Phase 2 (staged, disabled): further sources from the 2026-07-29 scoping round. +// +// Adding a source: pick the `kind` whose adapter fits (test the feed first), allowlist its +// image host in next.config.ts, and land it `enabled: false`. +// +// NOTE on virtual events: none of the Tribe orgs sets Tribe's `is_virtual` flag (it ships +// with the paid Virtual Events add-on) — it is false or absent on every event. They mark +// online events by registering a venue named "Online" / "Webinar" instead, which is why +// isOnlineVenueName drives event_type. Don't rely on `is_virtual` alone. +// +// centrecanada.org is healthy but currently returns total:0 (empty upcoming calendar) — +// a zero count from it is expected, not a fetch failure. +export const SOURCES: Source[] = [ + { slug: 'mosaic', name: 'MOSAIC', kind: 'tribe', host: 'mosaicbc.org', enabled: true }, + { + slug: 'burnaby-nh', + name: 'Burnaby Neighbourhood House', + kind: 'tribe', + host: 'burnabynh.ca', + enabled: true, + }, + { slug: 'success', name: 'S.U.C.C.E.S.S.', kind: 'tribe', host: 'successbc.ca', enabled: true }, + { + slug: 'centre-canada', + name: 'CentreCanada', + kind: 'tribe', + host: 'centrecanada.org', + enabled: true, + }, + { + slug: 'pirs', + name: 'Pacific Immigrant Resources Society', + kind: 'tribe', + host: 'pirs.bc.ca', + enabled: true, + }, + + // --- Phase 2, staged and NOT crawled until `enabled` flips (see Source.enabled) --- + // All are libraries, so `relevanceFilter` is on: their calendars are mostly storytimes + // and drop-in clubs, with settlement content a small minority. See lib/relevance.ts. + { + slug: 'westvan-library', + name: 'West Vancouver Memorial Library', + kind: 'tribe', + host: 'westvanlibrary.ca', + enabled: false, + relevanceFilter: true, + }, + { + slug: 'vpl', + name: 'Vancouver Public Library', + kind: 'bibliocommons', + host: 'vpl', // BiblioCommons tenant slug, not a hostname — see the adapter's caution + enabled: false, + relevanceFilter: true, + }, + // Burnaby Public Library is deliberately absent: its BiblioCommons tenant (`burnaby`) + // answers "The Events feature is not available", and bpl.bc.ca/events is a client- + // rendered SPA with nothing server-side to read. Note `bpl` on BiblioCommons is BOSTON + // Public Library — a healthy but entirely wrong feed. See adapters/bibliocommons.ts. +]; + +/** + * The sources a run actually crawls. Everything downstream — the fetch fan-out and the + * per-source counts in the response — is scoped to this, so a disabled source is inert + * rather than merely unreported. + */ +export const ACTIVE_SOURCES = SOURCES.filter((source) => source.enabled); + +/** Exhaustive over SourceKind, so a new kind without an adapter is a compile error. */ +export const ADAPTERS: Record = { + tribe: tribe.fetchEvents, + bibliocommons: bibliocommons.fetchEvents, +}; + +/** + * Build the per-run context. Computed once per run so every source is bounded by the same + * instant, and so the Pexels candidate cache is shared across sources (identical topic + * queries then cost a single API call). + */ +export function makeContext(now: Date = new Date()): AdapterContext { + const today = todayInTimezone(ORG_TIMEZONE, now); + const windowEnd = windowEndInTimezone(ORG_TIMEZONE, WINDOW_MONTHS, now); + return { + pexelsCache: new Map(), + // Deliberately loose: parsed as UTC while the date itself is on the source's calendar, + // so the per-row guard trails a server-side bound by a few hours. It exists to catch a + // source that ignores (or cannot express) an end bound, not to police the hour. + windowEndMs: Date.parse(`${windowEnd}T23:59:59Z`), + nowMs: now.getTime(), + today, + windowEnd, + }; +} diff --git a/supabase/functions/events-crawler/lib/text.ts b/supabase/functions/events-crawler/lib/text.ts new file mode 100644 index 0000000..f51d233 --- /dev/null +++ b/supabase/functions/events-crawler/lib/text.ts @@ -0,0 +1,93 @@ +// Dependency-free text helpers shared by every adapter. Source bodies are arbitrary +// third-party HTML, so these are deliberately defensive. + +import { MAX_DESCRIPTION_CHARS } from './constants.ts'; + +/** + * `String.fromCodePoint` throws RangeError for anything outside 0…0x10FFFF — a feed + * carrying `�` or `�` is enough. That throw would escape + * decodeEntities() through clean()/htmlToParagraphs() and the row mapper into the + * adapter's catch, which returns [] — so ONE malformed entity in ONE item would discard + * that source's whole batch. Out-of-range values therefore keep their literal source + * text (`raw`) rather than throwing or being silently dropped. + */ +function codePointOr(raw: string, n: number): string { + return Number.isInteger(n) && n >= 0 && n <= 0x10ffff ? String.fromCodePoint(n) : raw; +} + +export function decodeEntities(input: string): string { + return input + .replace(/&/g, '&') + .replace(/</g, '<') + .replace(/>/g, '>') + .replace(/"/g, '"') + .replace(/'/g, "'") + .replace(/�*39;/g, "'") + .replace(/�*27;/gi, "'") + .replace(/�*160;/g, ' ') + .replace(/ /g, ' ') + .replace(/–/g, '–') + .replace(/—/g, '—') + .replace(/‘|’/g, "'") + .replace(/“|”/g, '"') + .replace(/&#(\d+);/g, (m, n) => codePointOr(m, Number(n))) + .replace(/&#x([0-9a-f]+);/gi, (m, n) => codePointOr(m, parseInt(n, 16))); +} + +/** Decode entities, collapse whitespace, trim. For single-line fields (title, venue). */ +export function clean(value: unknown): string { + if (value === null || value === undefined || value === '') return ''; + return decodeEntities(String(value)).replace(/\s+/g, ' ').trim(); +} + +/** + * Convert an HTML fragment to plain text that PRESERVES paragraph breaks as `\n` (the + * event detail page splits description on `\n` to render paragraphs). Block-ish tags + * become newlines; remaining tags are stripped; entities decoded; runs of blank lines + * collapsed. Source bodies (esp. MOSAIC's Cvent form markup and BiblioCommons' + * accessibility boilerplate) carry heavy nesting, so we normalise aggressively. + */ +export function htmlToParagraphs(html: string | null | undefined): string { + if (!html) return ''; + const withBreaks = String(html) + .replace(/<\s*br\s*\/?\s*>/gi, '\n') + .replace(/<\/\s*(p|div|li|ul|ol|h[1-6]|tr|blockquote)\s*>/gi, '\n') + .replace(/<[^>]*>/g, ' '); + return decodeEntities(withBreaks) + .split('\n') + .map((line) => line.replace(/\s+/g, ' ').trim()) + .filter(Boolean) + .join('\n') + .slice(0, MAX_DESCRIPTION_CHARS); +} + +// Venue names that denote a virtual room rather than a physical place. Sources that lack +// a virtual flag (Tribe's `is_virtual` ships with a paid add-on) mark online events by +// registering a venue literally named "Online" or "Webinar", so the venue NAME is often +// the only signal — without this every webinar types as in-person. +const VIRTUAL_STRONG = new Set([ + 'online', 'virtual', 'webinar', 'zoom', 'remote', 'teleconference', 'livestream', + 'webex', 'teams', 'meet', +]); +const VIRTUAL_FILLER = new Set([ + 'event', 'events', 'meeting', 'session', 'workshop', 'only', 'platform', 'via', 'web', + 'google', 'ms', 'microsoft', 'stream', 'live', +]); + +/** + * True when the whole venue name is virtual vocabulary ("Online", "Webinar", + * "Online (Zoom)", "Virtual Event"). Matches the ENTIRE name rather than searching for a + * substring, so a physical venue that merely contains one of these words — "Online + * Learning Centre, 123 Main St" — is not misread as virtual. + */ +export function isOnlineVenueName(name: string): boolean { + const tokens = name + .toLowerCase() + .replace(/[^a-z0-9]+/g, ' ') + .trim() + .split(/\s+/) + .filter(Boolean); + if (tokens.length === 0) return false; + if (!tokens.every((t) => VIRTUAL_STRONG.has(t) || VIRTUAL_FILLER.has(t))) return false; + return tokens.some((t) => VIRTUAL_STRONG.has(t)); +} diff --git a/supabase/functions/events-crawler/lib/types.ts b/supabase/functions/events-crawler/lib/types.ts new file mode 100644 index 0000000..ed6aaa6 --- /dev/null +++ b/supabase/functions/events-crawler/lib/types.ts @@ -0,0 +1,107 @@ +// Shared shapes for the crawler registry, its adapters, and the rows they produce. + +/** + * Which adapter handles a source. Adding a value here means adding an entry to the + * ADAPTERS map in index.ts — the map is exhaustive over this union, so a missing + * implementation is a type error rather than a runtime surprise. + */ +export type SourceKind = 'tribe' | 'bibliocommons'; + +export interface Source { + /** Stored into `events.source` as `crawler:`. */ + slug: string; + /** Fallback hosted_by / display name when an event carries no organizer. */ + name: string; + /** Which adapter fetches this source. */ + kind: SourceKind; + /** Hostname for host-based sources; the library slug for BiblioCommons. */ + host: string; + /** + * Whether a run actually crawls this source. New sources land as `false`. + * + * The weekly cron is LIVE (20260724120000_events_crawler_cron.sql — pg_cron jobid 18, + * 'events-crawler-weekly', Mondays 14:00 UTC), so deploying this function is what puts + * a new source into production: the next run would auto-publish it into the shared + * events table the mobile app reads. Landing new sources disabled decouples the two, + * leaving activation as its own one-line, reviewable change — which needs Savar's + * sign-off, since it writes into shared mobile-facing data (CLAUDE.md, "Decision + * Authority"). + */ + enabled: boolean; + /** + * Apply the settlement-relevance filter to this source's titles (see lib/relevance.ts). + * + * Set on libraries and universities, whose calendars are mostly storytimes, teen + * gaming and academic seminars. NOT set on the settlement agencies, where everything + * published is already on-mission and filtering would only lose events. + */ + relevanceFilter?: boolean; + /** + * Location to use when the source's feed carries no location data at all. Only for + * feeds where the field genuinely does not exist — never to paper over a parse miss, + * since `events.location` is NOT NULL and a wrong location is worse than a dropped row. + */ + defaultLocation?: string; +} + +/** + * public.events.genre is a pre-existing shared column (default 'Uncategorized'). The + * mobile app types it as EventGenre and ships a genre chip filter that is currently + * commented out; the first five values below are exactly its enum, so nothing renames + * when Savar uncomments it — Language/Health/Family/Education are purely additive. + */ +export type EventGenre = + | 'Employment' + | 'Language' + | 'Housing' + | 'Finance' + | 'Documentation' + | 'Health' + | 'Family' + | 'Education' + | 'Socials' + | 'Uncategorized'; + +/** Exactly the insert shape for public.events. */ +export interface EventRow { + title: string; + description: string | null; + event_datetime: string; + event_end_datetime: string | null; + location: string; + event_type: 'in-person' | 'online' | 'hybrid'; + cover_photo_url: string | null; + external_link: string; + hosted_by: string | null; + address: string | null; + genre: EventGenre; + source: string; +} + +/** Per-run state threaded into every adapter. */ +export interface AdapterContext { + /** + * Pexels candidate-list memo, one per run and shared across sources, so identical + * topic queries cost a single API call. + */ + pexelsCache: Map>; + /** Far edge of the rolling window as epoch ms — the per-row backstop. */ + windowEndMs: number; + /** + * Run start as epoch ms — the near edge of the window. Adapters whose source has no + * server-side "from today" bound (BiblioCommons returns its entire calendar, past + * included) must apply this themselves, or already-finished events get ingested. + */ + nowMs: number; + /** Today as YYYY-MM-DD on the source's calendar. */ + today: string; + /** Window end as YYYY-MM-DD on the source's calendar. */ + windowEnd: string; +} + +/** + * An adapter turns one source into rows ready for insert. It is responsible for its own + * fetching, mapping, window filtering, relevance filtering and image resolution, and + * must never throw — a source that fails returns [] so the other sources still land. + */ +export type Adapter = (source: Source, ctx: AdapterContext) => Promise;