diff --git a/next.config.ts b/next.config.ts index 0885339..cef3fb6 100644 --- a/next.config.ts +++ b/next.config.ts @@ -44,6 +44,10 @@ const nextConfig: NextConfig = { { protocol: "https", hostname: "www.westvanlibrary.ca" }, // BiblioCommons serves each library's event covers off its own tenant subdomain. { protocol: "https", hostname: "vpl.bibliocommons.com" }, + // Capilano serves event images from /media/ on its own host. (NVCL needs no entry — + // its listing renders an icon font rather than photos, so covers always fall through + // to the Pexels/Unsplash tiers.) + { protocol: "https", hostname: "www.capilanou.ca" }, // SFU's LiveWhale calendar serves thumbnails off the calendar host itself. { protocol: "https", hostname: "events.sfu.ca" }, // Communico (NVDPL) hosts event images in a public S3 bucket. Pinned to the exact diff --git a/supabase/functions/events-crawler/adapters/capilano.ts b/supabase/functions/events-crawler/adapters/capilano.ts new file mode 100644 index 0000000..9acaa7d --- /dev/null +++ b/supabase/functions/events-crawler/adapters/capilano.ts @@ -0,0 +1,320 @@ +// Adapter: Capilano University (Terminalfour CMS, server-rendered HTML). +// +// Deferred from the 2026-07-29 round on the belief its listing carried no dates and would +// need a fetch per event. Re-checked 2026-08-02: not so. Each `li.event-item` carries the +// month, day, optional end date, optional times, venue, link and image inline, so there is +// no N+1. +// +// ONE PAGE ONLY, AND THAT IS A ROBOTS DECISION, NOT AN OVERSIGHT. +// capilanou.ca/robots.txt has a `User-agent: *` block disallowing `/*?search=*` and +// `/*&search=*`, and every calendar-navigation URL the page offers is of the form +// `?day=14&month=08&year=2026&search=day`. So the paginated/month views are off-limits to +// us and this adapter reads only the unparameterized listing. The practical ceiling is +// about ten upcoming items — do not "fix" a low count by adding query parameters. +// +// Expect a very low yield, and that is correct rather than broken. The listing is mostly +// the academic administrative calendar — fee deadlines, grade-submission dates, campus +// closures — which `relevanceFilter` is right to drop. Measured 2026-08-02: 1 of 10 items +// passed, "Fall 2026 New International Student Orientation", caught by relevance.ts's +// `international student` term. One genuinely on-mission event is the realistic yield here. +// +// MOST ITEMS HAVE NO TIME. 8 of those 10 are all-day administrative entries with no +// `datelisting` span at all — including the one relevant event. `events.event_datetime` is +// NOT NULL, so those are stored at ALL_DAY_HOUR local. That asserts an hour the source does +// not state, which is a real if small cost; the alternative is dropping every all-day item, +// which would have dropped the only event worth having. +// +// NO WEEKDAY, so the year cannot be resolved the way adapters/nvcl.ts does it. The listing +// is chronological instead, so the year is carried forward monotonically: start on the +// org's current year and roll to the next as soon as a month/day goes backwards relative to +// the previous row. That is what keeps a December-to-January listing from filing January in +// the past. + +import { FETCH_TIMEOUT_MS, MAX_PER_ORG, MAX_TITLE_CHARS, ORG_TIMEZONE, USER_AGENT } from '../lib/constants.ts'; +import { zonedWallClockToUtc } from '../lib/dates.ts'; +import { genreForEvent } from '../lib/genre.ts'; +import { resolveCover } from '../lib/images.ts'; +import { isSettlementRelevant } from '../lib/relevance.ts'; +import { clean, isOnlineVenueName } from '../lib/text.ts'; +import type { AdapterContext, EventRow, Source } from '../lib/types.ts'; + +/** Opens each event row in the listing. */ +const BLOCK_MARKER = '
  • '; +/** Start stamp. `date-stamp2`, when present, is the end of a multi-day event. */ +const START_STAMP_RE = + /
    \s*
    \s*([A-Za-z]+)\s*<\/div>\s*
    \s*(\d{1,2})\s*<\/div>/i; +const END_STAMP_RE = + /
    \s*
    \s*([A-Za-z]+)\s*<\/div>\s*
    \s*(\d{1,2})\s*<\/div>/i; +const TITLE_LINK_RE = /

    \s*]*>\s*([\s\S]*?)<\/a>/i; +/** Start time, and optionally an end time, e.g. "7:30 PM" - "9:30 PM". Absent on all-day items. */ +const TIMES_RE = + /\s*([^<]*?)\s*<\/span>(?:\s*-\s*\s*([^<]*?)\s*<\/span>)?/i; +/** Venue text sits in the anchor after the location icon, prefixed "Venues>". */ +const VENUE_RE = /fa-location-arrow[\s\S]{0,160}?]*>([\s\S]*?)<\/a>/i; +const VENUE_PREFIX_RE = /^Venues\s*>\s*/i; +const IMAGE_RE = /]*class="event-image"/i; + +const TIME_RE = /^(\d{1,2}):(\d{2})\s*([ap]\.?m\.?)$/i; + +const MONTHS: Record = { + jan: 1, feb: 2, mar: 3, apr: 4, may: 5, jun: 6, + jul: 7, aug: 8, sep: 9, oct: 10, nov: 11, dec: 12, +}; + +/** + * Local hour used for all-day items, which are the majority here. 09:00 rather than + * midnight: the row renders as a time in the UI, and "9:00 AM" reads as a business-day + * event where "12:00 AM" reads as a bug. + */ +const ALL_DAY_HOUR = 9; + +function to24Hour(hour12: number, meridiem: string): number { + const h = hour12 % 12; + return meridiem.toLowerCase().startsWith('p') ? h + 12 : h; +} + +/** "7:30 PM" → {hour, minute}, or null when the string isn't a time. */ +function parseTime(value: string | undefined): { hour: number; minute: number } | null { + if (!value) return null; + const m = value.trim().match(TIME_RE); + if (!m) return null; + return { hour: to24Hour(Number(m[1]), m[3]), minute: Number(m[2]) }; +} + +/** Per-layer tally of blocks the parse could not read at all. */ +interface StructuralMisses { + /** No title anchor, or an anchor with an empty title. */ + title: number; + /** Title found, but no start date stamp, or an unknown month name. */ + stamp: number; + /** Title and stamp found, but no venue text — and events.location is NOT NULL. */ + venue: number; +} + +interface Candidate { + title: string; + link: string; + startIso: string; + endIso: string | null; + location: string; + imageUrl: string | null; +} + +interface PageParse { + candidates: Candidate[]; + /** Blocks seen, before any filtering — 0 means the markup marker stopped matching. */ + blocks: number; + /** + * Structural rejections, split by layer. Deliberately excludes the relevance and window + * filters: those reject on merit, and dropping most of an academic calendar is their job, + * so counting them would make the page-level diagnostic fire on every healthy run. + */ + misses: StructuralMisses; +} + +async function fetchListing(source: Source): Promise { + // No query string, by robots policy — see the header. + const url = `https://${source.host}/about-capu/get-to-know-us/events/`; + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), FETCH_TIMEOUT_MS); + try { + const res = await fetch(url, { + signal: controller.signal, + headers: { 'User-Agent': USER_AGENT, Accept: 'text/html' }, + }); + if (!res.ok) { + console.error(`events-crawler: ${source.slug} listing returned HTTP ${res.status}`); + return null; + } + return await res.text(); + } catch (error) { + console.error(`events-crawler: ${source.slug} listing failed:`, error); + return null; + } finally { + clearTimeout(timer); + } +} + +function parseListing(html: string, source: Source, ctx: AdapterContext): PageParse { + const chunks = html.split(BLOCK_MARKER).slice(1); + const candidates: Candidate[] = []; + const misses: StructuralMisses = { title: 0, stamp: 0, venue: 0 }; + + // Year carried forward across the chronological listing — see the header. + let year = Number( + new Intl.DateTimeFormat('en-CA', { timeZone: ORG_TIMEZONE, year: 'numeric' }) + .format(new Date(ctx.nowMs)), + ); + let prevMonth = 0; + let prevDay = 0; + + for (const chunk of chunks) { + const titleMatch = chunk.match(TITLE_LINK_RE); + if (!titleMatch) { + misses.title++; + continue; + } + const href = titleMatch[1]; + // Filter on the FULL cleaned title, then truncate for storage. + const fullTitle = clean(titleMatch[2]); + if (!fullTitle) { + misses.title++; + continue; + } + + const startStamp = chunk.match(START_STAMP_RE); + const month = startStamp ? MONTHS[startStamp[1].slice(0, 3).toLowerCase()] : undefined; + if (!startStamp || !month) { + misses.stamp++; + continue; + } + const day = Number(startStamp[2]); + + // Roll the year forward the moment the listing wraps past December. + if (prevMonth && (month < prevMonth || (month === prevMonth && day < prevDay))) year++; + prevMonth = month; + prevDay = day; + + const startTime = parseTime(chunk.match(TIMES_RE)?.[1]); + const startIso = zonedWallClockToUtc( + ORG_TIMEZONE, + year, + month, + day, + startTime?.hour ?? ALL_DAY_HOUR, + startTime?.minute ?? 0, + ); + if (!startIso) { + // Not a real date on the calendar — a malformed stamp, not a filtered row. + misses.stamp++; + continue; + } + const startMs = Date.parse(startIso); + + // Venue is read here, BEFORE the filters, even though it is only needed for rows that + // survive them. Reading it after would put a structural failure downstream of a + // merit-based one: if VENUE_RE stopped matching, every row would drop at a `continue` + // that no counter reaches, and the source would go silently to zero. Keeping the whole + // extraction chain — title, stamp, venue — ahead of the filters means one comparison + // (`structural === blocks`) covers all of it. The cost is a regex on rows about to be + // discarded, over roughly ten blocks. + const venueMatch = chunk.match(VENUE_RE); + const location = clean(venueMatch?.[1]).replace(VENUE_PREFIX_RE, '').trim(); + if (!location) { + // events.location is NOT NULL, so a row without one cannot be stored at all. + misses.venue++; + continue; + } + + // Window before relevance, so both filters see every parsed row. + if (source.relevanceFilter && !isSettlementRelevant(fullTitle)) continue; + if (startMs < ctx.nowMs || startMs > ctx.windowEndMs) continue; + + // End date and end time are independent: a multi-day item has date-stamp2 but often no + // times, and a single-day concert has an end time but no second stamp. + const endStamp = chunk.match(END_STAMP_RE); + const endMonth = endStamp ? MONTHS[endStamp[1].slice(0, 3).toLowerCase()] : undefined; + const endTime = parseTime(chunk.match(TIMES_RE)?.[2]); + let endIso: string | null = null; + if (endStamp && endMonth) { + // A multi-day range that wraps the year end lands in the following year. + const endYear = endMonth < month ? year + 1 : year; + endIso = zonedWallClockToUtc( + ORG_TIMEZONE, + endYear, + endMonth, + Number(endStamp[2]), + endTime?.hour ?? ALL_DAY_HOUR, + endTime?.minute ?? 0, + ); + } else if (endTime) { + endIso = zonedWallClockToUtc(ORG_TIMEZONE, year, month, day, endTime.hour, endTime.minute); + } + // Never store an inverted range; the listing gives no signal for a past-midnight end. + if (endIso && Date.parse(endIso) <= startMs) endIso = null; + + const imageMatch = chunk.match(IMAGE_RE); + const rawImage = imageMatch?.[1]; + + candidates.push({ + title: fullTitle.slice(0, MAX_TITLE_CHARS), + link: href.startsWith('http') ? href : `https://${source.host}${href}`, + startIso, + endIso, + location, + imageUrl: rawImage + ? (rawImage.startsWith('http') ? rawImage : `https://${source.host}${rawImage}`) + : null, + }); + } + + return { candidates, blocks: chunks.length, misses }; +} + +export async function fetchEvents(source: Source, ctx: AdapterContext): Promise { + const html = await fetchListing(source); + if (html === null) return []; + + const parsed = parseListing(html, source, ctx); + + if (parsed.blocks === 0) { + console.error( + `events-crawler: ${source.slug} found no "${BLOCK_MARKER}" blocks — the listing markup ` + + `has probably changed`, + ); + return []; + } + // Blocks present but none survived extraction: the marker still matches while something + // inside it moved — the title anchor, the date stamp, or the venue. A source that returns + // nothing because its markup shifted otherwise looks exactly like a university with + // nothing on. + const m = parsed.misses; + if (m.title + m.stamp + m.venue === parsed.blocks) { + console.error( + `events-crawler: ${source.slug} matched ${parsed.blocks} block(s) but extracted none ` + + `of them (title ${m.title}, date stamp ${m.stamp}, venue ${m.venue}) — the row ` + + `markup has probably changed`, + ); + } + + // Dedupe by link. Defensive here rather than load-bearing: this listing showed 10 distinct + // links in 10 blocks, with no repeated featured section of the kind adapters/nvcl.ts has to + // handle. Kept so a future recurring-event grouping cannot produce a duplicate-key insert. + const byLink = new Map(); + for (const c of parsed.candidates) { + if (!byLink.has(c.link)) byLink.set(c.link, c); + } + const deduped = [...byLink.values()]; + deduped.sort((a, b) => a.startIso.localeCompare(b.startIso)); + const selected = deduped.slice(0, MAX_PER_ORG); + + console.log( + `events-crawler: ${source.slug} parsed ${parsed.blocks} block(s), ` + + `${parsed.candidates.length} in-window + relevant (${deduped.length} unique), ` + + `taking soonest ${selected.length}`, + ); + + // allSettled so one failed cover lookup can't reject the batch and discard the source. + const settled = await Promise.allSettled( + selected.map(async (c): Promise => ({ + title: c.title, + description: null, // the listing carries no summary; the detail page is a separate fetch + event_datetime: c.startIso, + event_end_datetime: c.endIso, + location: c.location, + event_type: isOnlineVenueName(c.location) ? 'online' : 'in-person', + cover_photo_url: await resolveCover(c.imageUrl, c.title, c.link, ctx.pexelsCache), + external_link: c.link, + hosted_by: source.name, + address: null, + genre: genreForEvent(c.title, null), + source: `crawler:${source.slug}`, + })), + ); + const rows: EventRow[] = []; + for (const result of settled) { + if (result.status === 'fulfilled') rows.push(result.value); + else console.error(`events-crawler: ${source.slug} row failed:`, result.reason); + } + return rows; +} diff --git a/supabase/functions/events-crawler/lib/sources.ts b/supabase/functions/events-crawler/lib/sources.ts index 5c0d3be..5171a8b 100644 --- a/supabase/functions/events-crawler/lib/sources.ts +++ b/supabase/functions/events-crawler/lib/sources.ts @@ -5,6 +5,7 @@ // here. Keeping one definition means the preview and a real run cannot drift. import * as bibliocommons from '../adapters/bibliocommons.ts'; +import * as capilano from '../adapters/capilano.ts'; import * as communico from '../adapters/communico.ts'; import * as livewhale from '../adapters/livewhale.ts'; import * as nvcl from '../adapters/nvcl.ts'; @@ -128,6 +129,19 @@ export const SOURCES: Source[] = [ // The listing's venue slot is present but empty on every row — see adapters/nvcl.ts. defaultLocation: 'North Vancouver City Library', }, + { + // Capped at ~10 items by robots.txt, which disallows the `?…search=…` URLs every + // calendar-navigation link uses — so a low count here is the ceiling, not a failure. + // Its calendar is largely academic administrivia (fee deadlines, closures), so expect + // roughly one on-mission event: measured 1 of 10 on 2026-08-02, the Fall New + // International Student Orientation. See adapters/capilano.ts. + slug: 'capilano', + name: 'Capilano University', + kind: 'capilano', + host: 'www.capilanou.ca', + enabled: false, + relevanceFilter: true, + }, ]; /** @@ -145,6 +159,7 @@ export const ADAPTERS: Record = { communico: communico.fetchEvents, 'surrey-drupal': surrey.fetchEvents, 'nvcl-drupal': nvcl.fetchEvents, + capilano: capilano.fetchEvents, }; /** diff --git a/supabase/functions/events-crawler/lib/text.ts b/supabase/functions/events-crawler/lib/text.ts index f51d233..a1662f5 100644 --- a/supabase/functions/events-crawler/lib/text.ts +++ b/supabase/functions/events-crawler/lib/text.ts @@ -26,6 +26,14 @@ export function decodeEntities(input: string): string { .replace(/�*27;/gi, "'") .replace(/�*160;/g, ' ') .replace(/ /g, ' ') + // Named forms of the punctuation already handled numerically below. Capilano publishes + // "Last Day of Classes for Summer 2026 – Session II", which without these lands in + // a title verbatim; the numeric branches never see it because it isn't numeric. + .replace(/–/g, '–') + .replace(/—/g, '—') + .replace(/‘|’/g, "'") + .replace(/“|”/g, '"') + .replace(/…/g, '…') .replace(/–/g, '–') .replace(/—/g, '—') .replace(/‘|’/g, "'") diff --git a/supabase/functions/events-crawler/lib/types.ts b/supabase/functions/events-crawler/lib/types.ts index 4b1b2e7..5bed1b1 100644 --- a/supabase/functions/events-crawler/lib/types.ts +++ b/supabase/functions/events-crawler/lib/types.ts @@ -11,7 +11,8 @@ export type SourceKind = | 'livewhale' | 'communico' | 'surrey-drupal' - | 'nvcl-drupal'; + | 'nvcl-drupal' + | 'capilano'; export interface Source { /** Stored into `events.source` as `crawler:`. */