From 9b3ea7f9a9fdcc3c64c66c3688534386c31f3925 Mon Sep 17 00:00:00 2001 From: ltanafranca1004 Date: Sun, 2 Aug 2026 16:13:34 -0700 Subject: [PATCH 1/4] feat(events-crawler): North Vancouver City Library adapter, staged disabled MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit First of the orgs deferred from the 2026-07-29 round. The deferral assumed the listing carried no dates and would need a fetch per event; re-checking the live feed showed otherwise — date and time are both in the listing, so there is no N+1. NVCL is the second library whose BiblioCommons tenant is a dead end. Unlike the bpl/Boston trap, `nvcl` really is North Vancouver City Library — but the gateway answers 403 "The Events feature is not available", exactly as Burnaby does. Verified before writing a line, and recorded in the adapter header so nobody re-tests it. The Drupal listing is the only readable surface. The listing carries no year. Every row reads "Tuesday, August 4, 10:30 am to 11:00 am" — 0 of 45 sampled rows across three pages carried one. That matters because the paginated list runs chronologically past the year boundary: page 37 of 38 lists April-June, i.e. the following year, which "assume current year" would file eight months in the past. So the year is derived and then *checked*: each candidate year's weekday is compared against the weekday the page printed, and a row matching neither is skipped rather than guessed at. Two more things the feed does not have. There is no location — the venue slot exists in the markup but is empty on every row sampled — so the source supplies defaultLocation, and the adapter refuses to run without one rather than inserting a wrong place into a NOT NULL column. And there are no images, only an icon font, so covers always fall through to the Pexels/Unsplash tiers. Dedupe-by-link is load-bearing here rather than defensive: every page repeats the same five featured rows above its paginated section, so without it a 24-page walk returns those five 24 times. Safe as an identity — across 45 sampled rows no slug appeared with two different dates, so the slug identifies the occurrence and matches events_external_link_key. Carries forward both parse-failure guards from surrey.ts (page 0 with no blocks, page 0 where every title misses), since this is the same class of scraper. KNOWN: with relevanceFilter on, this source currently yields ZERO rows — see the PR for the evidence and the decision it needs. Landing disabled either way. Verified with the real harness: deno check clean; dryrun --source nvcl parses 25 well-formed rows with the filter off (correct dates, end times, dedupe, covers) and 0 with it on. tsc and lint clean. Co-Authored-By: Claude Opus 5 (1M context) --- .../functions/events-crawler/adapters/nvcl.ts | 325 ++++++++++++++++++ .../functions/events-crawler/lib/sources.ts | 20 ++ .../functions/events-crawler/lib/types.ts | 3 +- 3 files changed, 347 insertions(+), 1 deletion(-) create mode 100644 supabase/functions/events-crawler/adapters/nvcl.ts diff --git a/supabase/functions/events-crawler/adapters/nvcl.ts b/supabase/functions/events-crawler/adapters/nvcl.ts new file mode 100644 index 0000000..b6802e3 --- /dev/null +++ b/supabase/functions/events-crawler/adapters/nvcl.ts @@ -0,0 +1,325 @@ +// Adapter: North Vancouver City Library (Drupal, server-rendered HTML). +// +// The second HTML-scraping adapter, and the second library whose BiblioCommons tenant is a +// dead end. `nvcl` on BiblioCommons *is* North Vancouver City Library — the identity is +// right, unlike the `bpl`/Boston trap documented in adapters/bibliocommons.ts — but the +// gateway answers 403 "The Events feature is not available at North Vancouver City +// Library", exactly as Burnaby's does. Verified before writing this, so the next person +// doesn't re-test it. The Drupal listing is the only readable surface. +// +// It earns the scraping cost: NVCL runs settlement-adjacent programming (conversation +// circles, newcomer drop-ins, the Open Door community hub) alongside the usual storytimes, +// which is what `relevanceFilter` is for. +// +// THE LISTING CARRIES NO YEAR. Every row reads "Tuesday, August 4, 10:30 am to 11:00 am" — +// weekday, month, day, times, and nothing else (verified: 0 of 45 sampled rows across three +// pages carried a year). That matters more than it looks, because the paginated list runs +// chronologically straight past the year boundary: page 37 of 38 lists April-June, i.e. the +// *following* year. Guessing "current year" would file those ~8 months in the past. +// +// So the year is *derived and then checked*, not assumed: for each candidate year the +// weekday of the resulting date is compared against the weekday the page printed, and the +// candidate that matches wins. A row whose weekday matches neither candidate is skipped +// rather than guessed at — a wrong date is worse than a missing event. +// +// NO LOCATION FIELD. The slot exists in the markup (a second

) +// but is empty on every row sampled, so the source supplies `defaultLocation`. Titles +// routinely name the venue anyway ("Outdoor storytime at Semisch Park", "Book Bike at +// MONOVA"), so the specific place is not lost, just not machine-readable. +// +// NO IMAGES either — the listing renders an icon font per event, not a photo, so covers +// always fall through to the Pexels/Unsplash tiers. +// +// Pagination is `?page=N`, 0-based and chronological, so the walk can stop once a page runs +// past the window. Every page additionally repeats the same five "featured" rows before its +// paginated section, which is why dedupe-by-link is load-bearing rather than defensive: +// without it a 20-page walk would return the same five events 20 times. + +import { FETCH_TIMEOUT_MS, MAX_PER_ORG, MAX_TITLE_CHARS, ORG_TIMEZONE, USER_AGENT } from '../lib/constants.ts'; +import { zonedWallClockToUtc } from '../lib/dates.ts'; +import { genreForEvent } from '../lib/genre.ts'; +import { resolveCover } from '../lib/images.ts'; +import { isSettlementRelevant } from '../lib/relevance.ts'; +import { clean, isOnlineVenueName } from '../lib/text.ts'; +import type { AdapterContext, EventRow, Source } from '../lib/types.ts'; + +/** Opens each event row in the Drupal listing. */ +const BLOCK_MARKER = '

]*>([\s\S]*?)<\/a>/i; +/** The first bold paragraph in the block is the when-line; the second is the empty venue slot. */ +const WHEN_RE = /

\s*([\s\S]*?)<\/p>/i; + +// "Tuesday, August 4, 10:30 am to 11:00 am" — the end time is optional. +const WHEN_PARSE_RE = + /^([A-Za-z]+),\s*([A-Za-z]+)\s+(\d{1,2}),\s*(\d{1,2}):(\d{2})\s*([ap]m)(?:\s*to\s*(\d{1,2}):(\d{2})\s*([ap]m))?/i; + +const MONTHS: Record = { + jan: 1, feb: 2, mar: 3, apr: 4, may: 5, jun: 6, + jul: 7, aug: 8, sep: 9, oct: 10, nov: 11, dec: 12, +}; + +const WEEKDAYS: Record = { + sun: 0, mon: 1, tue: 2, wed: 3, thu: 4, fri: 5, sat: 6, +}; + +/** Pages fetched per round before the stop conditions are re-checked. */ +const PAGE_CONCURRENCY = 4; +/** + * Hard bound on pages walked. The pager advertises 38 pages of 15; the 4-month window is + * reached well before that, so this is a backstop for a listing that grows or stops being + * chronological, not the expected exit. Hitting it is logged, never silent. + */ +const MAX_PAGES = 24; + +function to24Hour(hour12: number, meridiem: string): number { + const h = hour12 % 12; + return meridiem.toLowerCase() === 'pm' ? h + 12 : h; +} + +/** The weekday of a Y-M-D as rendered on the org's calendar, 0=Sunday. */ +function weekdayInTimezone(timeZone: string, iso: string): number | null { + const name = new Intl.DateTimeFormat('en-US', { timeZone, weekday: 'short' }) + .format(new Date(iso)) + .slice(0, 3) + .toLowerCase(); + const day = WEEKDAYS[name]; + return day === undefined ? null : day; +} + +interface ParsedWhen { + startIso: string; + endIso: string | null; +} + +/** + * Parse the when-line into UTC instants, resolving the missing year against the printed + * weekday. Returns null when the line doesn't match, the month is unknown, or no candidate + * year reproduces the weekday the page printed. + */ +function parseWhen(text: string, timeZone: string, nowMs: number): ParsedWhen | null { + const m = text.trim().match(WHEN_PARSE_RE); + if (!m) return null; + + const month = MONTHS[m[2].slice(0, 3).toLowerCase()]; + const printedWeekday = WEEKDAYS[m[1].slice(0, 3).toLowerCase()]; + if (!month || printedWeekday === undefined) return null; + + const day = Number(m[3]); + const startHour = to24Hour(Number(m[4]), m[6]); + const startMinute = Number(m[5]); + + // The listing only ever runs forwards from today, so the year is either the current one + // on the org's calendar or the next. Try both and keep the one whose weekday matches what + // the page printed — that is what disambiguates a page-37 "April 15" from this April. + const currentYear = Number( + new Intl.DateTimeFormat('en-CA', { timeZone, year: 'numeric' }).format(new Date(nowMs)), + ); + for (const year of [currentYear, currentYear + 1]) { + const startIso = zonedWallClockToUtc(timeZone, year, month, day, startHour, startMinute); + if (!startIso) continue; // not a real date on the calendar (e.g. Feb 30) + if (weekdayInTimezone(timeZone, startIso) !== printedWeekday) continue; + + let endIso: string | null = null; + if (m[7]) { + const endHour = to24Hour(Number(m[7]), m[9]); + const candidate = zonedWallClockToUtc(timeZone, year, month, day, endHour, Number(m[8])); + // An end before the start means the event runs past midnight; the listing gives no + // end date to confirm that, so drop the end rather than store an inverted range. + if (candidate && Date.parse(candidate) > Date.parse(startIso)) endIso = candidate; + } + return { startIso, endIso }; + } + return null; +} + +async function fetchPage(source: Source, page: number): Promise { + const url = `https://${source.host}/events?page=${page}`; + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), FETCH_TIMEOUT_MS); + try { + const res = await fetch(url, { + signal: controller.signal, + headers: { 'User-Agent': USER_AGENT, Accept: 'text/html' }, + }); + if (!res.ok) { + console.error(`events-crawler: ${source.slug} page ${page} returned HTTP ${res.status}`); + return null; + } + return await res.text(); + } catch (error) { + console.error(`events-crawler: ${source.slug} page ${page} failed:`, error); + return null; + } finally { + clearTimeout(timer); + } +} + +interface Candidate { + title: string; + link: string; + startIso: string; + endIso: string | null; +} + +interface PageParse { + candidates: Candidate[]; + /** Blocks seen, before any filtering — 0 means the listing has run out. */ + blocks: number; + /** Blocks whose title link did not match, out of `blocks`. */ + titleMisses: number; + /** Latest start on the page, to decide whether the window has been walked past. */ + maxStartMs: number; +} + +function parsePage(html: string, source: Source, ctx: AdapterContext): PageParse { + const chunks = html.split(BLOCK_MARKER).slice(1); + const candidates: Candidate[] = []; + let maxStartMs = 0; + let titleMisses = 0; + + for (const chunk of chunks) { + const titleMatch = chunk.match(TITLE_LINK_RE); + if (!titleMatch) { + titleMisses++; + continue; + } + const href = titleMatch[1]; + // Filter on the FULL cleaned title, then truncate for storage. + const fullTitle = clean(titleMatch[2]); + if (!fullTitle) continue; + + const whenMatch = chunk.match(WHEN_RE); + if (!whenMatch) continue; + const parsed = parseWhen(clean(whenMatch[1]), ORG_TIMEZONE, ctx.nowMs); + if (!parsed) continue; + + const startMs = Date.parse(parsed.startIso); + if (startMs > maxStartMs) maxStartMs = startMs; + + // Track the window before relevance, so the walk can stop on dates even when a page + // happens to contain nothing relevant. + if (source.relevanceFilter && !isSettlementRelevant(fullTitle)) continue; + if (startMs < ctx.nowMs || startMs > ctx.windowEndMs) continue; + + candidates.push({ + title: fullTitle.slice(0, MAX_TITLE_CHARS), + link: `https://${source.host}${href}`, + startIso: parsed.startIso, + endIso: parsed.endIso, + }); + } + + return { candidates, blocks: chunks.length, titleMisses, maxStartMs }; +} + +export async function fetchEvents(source: Source, ctx: AdapterContext): Promise { + // events.location is NOT NULL and this feed has no location field at all, so a source + // without defaultLocation would produce rows that cannot be inserted. Fail loudly here + // rather than at the insert. + const location = source.defaultLocation; + if (!location) { + console.error( + `events-crawler: ${source.slug} has no defaultLocation, but its listing carries no ` + + `location field — skipping the source rather than inserting a wrong place.`, + ); + return []; + } + + const candidates: Candidate[] = []; + let pagesWalked = 0; + let stoppedEarly = false; + let exhausted = false; + + for (let page = 0; page < MAX_PAGES; page += PAGE_CONCURRENCY) { + const batch: number[] = []; + for (let p = page; p < Math.min(page + PAGE_CONCURRENCY, MAX_PAGES); p++) batch.push(p); + const pages = await Promise.all(batch.map((p) => fetchPage(source, p))); + + let pastWindow = false; + for (const html of pages) { + pagesWalked++; + if (html === null) continue; + const parsed = parsePage(html, source, ctx); + if (parsed.blocks === 0) { + // Either the listing ran out, or the markup changed under us. Page 0 is never + // legitimately empty for a live listing, so that case is a parse failure and is + // logged as one. + if (pagesWalked === 1) { + console.error( + `events-crawler: ${source.slug} found no "${BLOCK_MARKER}" blocks on page 0 — ` + + `the listing markup has probably changed`, + ); + } + exhausted = true; + continue; + } + // Blocks present but every title missed: the marker still matches while the anchor + // markup moved. Same failure mode one layer down, and just as silent without this. + if (pagesWalked === 1 && parsed.titleMisses === parsed.blocks) { + console.error( + `events-crawler: ${source.slug} matched ${parsed.blocks} block(s) on page 0 but ` + + `extracted no title from any of them — the row markup has probably changed`, + ); + } + candidates.push(...parsed.candidates); + if (parsed.maxStartMs > ctx.windowEndMs) pastWindow = true; + } + + if (exhausted || pastWindow || candidates.length >= MAX_PER_ORG) { + stoppedEarly = true; + break; + } + } + + if (!stoppedEarly && pagesWalked >= MAX_PAGES) { + console.warn( + `events-crawler: ${source.slug} stopped at the ${MAX_PAGES}-page cap with ` + + `${candidates.length} candidate(s) — listings beyond it are the furthest out and ` + + `will be picked up on a later run.`, + ); + } + + // Dedupe by link. Not defensive: every page repeats the same five featured rows above its + // paginated section, so without this a multi-page walk returns them once per page. + // Verified safe as an identity — across 45 sampled rows no slug appeared with two + // different dates, so the slug identifies the occurrence and matches the + // events_external_link_key unique index. + const byLink = new Map(); + for (const c of candidates) { + if (!byLink.has(c.link)) byLink.set(c.link, c); + } + const deduped = [...byLink.values()]; + deduped.sort((a, b) => a.startIso.localeCompare(b.startIso)); + const selected = deduped.slice(0, MAX_PER_ORG); + + console.log( + `events-crawler: ${source.slug} walked ${pagesWalked} page(s), ${candidates.length} ` + + `in-window + relevant (${deduped.length} unique), taking soonest ${selected.length}`, + ); + + // allSettled so one failed cover lookup can't reject the batch and discard the source. + const settled = await Promise.allSettled( + selected.map(async (c): Promise => ({ + title: c.title, + description: null, // the listing carries no summary; the detail page is a separate fetch + event_datetime: c.startIso, + event_end_datetime: c.endIso, + location, + event_type: isOnlineVenueName(location) ? 'online' : 'in-person', + // The listing renders an icon font rather than a photo, so this always falls through + // to the Pexels topic photo and then the deterministic Unsplash pool. + cover_photo_url: await resolveCover(null, c.title, c.link, ctx.pexelsCache), + external_link: c.link, + hosted_by: source.name, + address: null, + genre: genreForEvent(c.title, null), + source: `crawler:${source.slug}`, + })), + ); + const rows: EventRow[] = []; + for (const result of settled) { + if (result.status === 'fulfilled') rows.push(result.value); + else console.error(`events-crawler: ${source.slug} row failed:`, result.reason); + } + return rows; +} diff --git a/supabase/functions/events-crawler/lib/sources.ts b/supabase/functions/events-crawler/lib/sources.ts index 4f36328..5c0d3be 100644 --- a/supabase/functions/events-crawler/lib/sources.ts +++ b/supabase/functions/events-crawler/lib/sources.ts @@ -7,6 +7,7 @@ import * as bibliocommons from '../adapters/bibliocommons.ts'; import * as communico from '../adapters/communico.ts'; import * as livewhale from '../adapters/livewhale.ts'; +import * as nvcl from '../adapters/nvcl.ts'; import * as surrey from '../adapters/surrey.ts'; import * as tribe from '../adapters/tribe.ts'; import { ORG_TIMEZONE, WINDOW_MONTHS } from './constants.ts'; @@ -109,6 +110,24 @@ export const SOURCES: Source[] = [ enabled: true, relevanceFilter: true, }, + + // --- Phase 3, staged and NOT crawled until `enabled` flips (see Source.enabled) --- + // Deferred from the 2026-07-29 round on the belief that their listings carried no dates + // and would need a per-event fetch. Re-checked 2026-08-02: not so — the date is in the + // listing, so no N+1. See each adapter's header for what the re-check actually found. + { + // NVCL, not NVDPL: different library system, different feed. Its BiblioCommons tenant + // is correctly named but answers 403 "The Events feature is not available", the same + // dead end as Burnaby — hence a Drupal scraper rather than the bibliocommons adapter. + slug: 'nvcl', + name: 'North Vancouver City Library', + kind: 'nvcl-drupal', + host: 'www.nvcl.ca', + enabled: false, + relevanceFilter: true, + // The listing's venue slot is present but empty on every row — see adapters/nvcl.ts. + defaultLocation: 'North Vancouver City Library', + }, ]; /** @@ -125,6 +144,7 @@ export const ADAPTERS: Record = { livewhale: livewhale.fetchEvents, communico: communico.fetchEvents, 'surrey-drupal': surrey.fetchEvents, + 'nvcl-drupal': nvcl.fetchEvents, }; /** diff --git a/supabase/functions/events-crawler/lib/types.ts b/supabase/functions/events-crawler/lib/types.ts index b6110b6..4b1b2e7 100644 --- a/supabase/functions/events-crawler/lib/types.ts +++ b/supabase/functions/events-crawler/lib/types.ts @@ -10,7 +10,8 @@ export type SourceKind = | 'bibliocommons' | 'livewhale' | 'communico' - | 'surrey-drupal'; + | 'surrey-drupal' + | 'nvcl-drupal'; export interface Source { /** Stored into `events.source` as `crawler:`. */ From 96aa3220382c14dfd68c731ee1f9784128786c75 Mon Sep 17 00:00:00 2001 From: ltanafranca1004 Date: Sun, 2 Aug 2026 16:20:05 -0700 Subject: [PATCH 2/4] fix(events-crawler): stop NVCL's walk on the last row, not the page maximum MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses the CodeRabbit finding, which is a real latent truncation bug. parsePage took the maximum start across every block on the page, but each page is "five featured rows, then the chronological section". The featured rows are curated, so one can sit arbitrarily far out — a save-the-date. That maximum would then exceed the window on page 0, set pastWindow on the very first batch, and stop the walk after 4 pages while in-window listings remained. Not reachable today (the featured rows are currently Aug 4-5, well inside the window) but entirely reachable by an editorial choice on the library's side, with no signal that it had happened. Took a different fix from the one sketched. Requiring a whole batch to agree before stopping does not work here: the featured block repeats on EVERY page, so a distant featured row makes every page in the batch report the same past-the-window maximum and the walk still truncates. Excluding "the leading five" would work but hardcodes a count the markup never promises. Using the LAST block in document order sidesteps both. The chronological section always trails the featured block, so the final block is the furthest-out entry the paginated list has actually reached — which is the thing the walk wants to compare against the window — and it needs no guess about how many featured rows exist. Verified against the live feed: with the relevance filter off the walk still ends early and correctly, taking 4 pages, 58 candidates, 38 unique, soonest 25. The 20 dropped duplicates are exactly the 5 featured rows repeated across those 4 pages, which is the repetition this finding is about. deno check clean. Co-Authored-By: Claude Opus 5 (1M context) --- .../functions/events-crawler/adapters/nvcl.ts | 26 ++++++++++++++----- 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/supabase/functions/events-crawler/adapters/nvcl.ts b/supabase/functions/events-crawler/adapters/nvcl.ts index b6802e3..71cdca8 100644 --- a/supabase/functions/events-crawler/adapters/nvcl.ts +++ b/supabase/functions/events-crawler/adapters/nvcl.ts @@ -167,14 +167,28 @@ interface PageParse { blocks: number; /** Blocks whose title link did not match, out of `blocks`. */ titleMisses: number; - /** Latest start on the page, to decide whether the window has been walked past. */ - maxStartMs: number; + /** + * Start of the LAST block on the page, in document order — the stop signal. + * + * Deliberately not the maximum. Each page is "five featured rows, then the chronological + * section", and the featured rows are curated, so one of them can sit arbitrarily far + * out (a save-the-date). A page maximum would then exceed the window on page 0 and + * truncate the walk after the first batch, silently dropping in-window listings — and + * because the featured block repeats on EVERY page, requiring a whole batch to agree + * wouldn't help either; every page would report the same distant maximum. + * + * The last block always belongs to the chronological section, so its start is the + * furthest-out entry the paginated list has actually reached. That is the thing the walk + * wants to compare against the window, and it needs no guess about how many featured + * rows there are. + */ + lastStartMs: number; } function parsePage(html: string, source: Source, ctx: AdapterContext): PageParse { const chunks = html.split(BLOCK_MARKER).slice(1); const candidates: Candidate[] = []; - let maxStartMs = 0; + let lastStartMs = 0; let titleMisses = 0; for (const chunk of chunks) { @@ -194,7 +208,7 @@ function parsePage(html: string, source: Source, ctx: AdapterContext): PageParse if (!parsed) continue; const startMs = Date.parse(parsed.startIso); - if (startMs > maxStartMs) maxStartMs = startMs; + lastStartMs = startMs; // Track the window before relevance, so the walk can stop on dates even when a page // happens to contain nothing relevant. @@ -209,7 +223,7 @@ function parsePage(html: string, source: Source, ctx: AdapterContext): PageParse }); } - return { candidates, blocks: chunks.length, titleMisses, maxStartMs }; + return { candidates, blocks: chunks.length, titleMisses, lastStartMs }; } export async function fetchEvents(source: Source, ctx: AdapterContext): Promise { @@ -262,7 +276,7 @@ export async function fetchEvents(source: Source, ctx: AdapterContext): Promise< ); } candidates.push(...parsed.candidates); - if (parsed.maxStartMs > ctx.windowEndMs) pastWindow = true; + if (parsed.lastStartMs > ctx.windowEndMs) pastWindow = true; } if (exhausted || pastWindow || candidates.length >= MAX_PER_ORG) { From eed0f64a66f4a0c3a842ce839501a7c82b9f2d82 Mon Sep 17 00:00:00 2001 From: ltanafranca1004 Date: Sun, 2 Aug 2026 17:25:45 -0700 Subject: [PATCH 3/4] fix(events-crawler): count NVCL's early-stop against unique links, not raw hits MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Second CodeRabbit finding on this adapter, and valid for the same underlying reason as the first: the featured block repeats on every page, so any running total that ignores it means something other than what it says. The MAX_PER_ORG early-stop compared candidates.length, which counted the featured rows once per page. With the relevance filter on — the shipped configuration — a source whose relevant rows are mostly featured ones would reach 25 in raw count while holding a handful of distinct events, stop there, and return that handful with in-window listings still unread. The observed run hid this (58 raw / 38 unique, both above the cap) but the shape of the bug does not depend on that mix. Deduping during the walk rather than after it fixes the count at its source and removes the separate post-loop pass: byLink.size is now what the stop condition tests, what the page-cap warning reports, and what gets sorted and sliced. A raw counter is kept purely for the log line, where "58 in-window + relevant (38 unique)" is genuinely more informative than either number alone. Verified against the live feed. Filter off: 4 pages, 58 raw, 38 unique, soonest 25 — identical to before, so the happy path is unchanged. Filter on: still 0, now reported as "0 unique candidate(s)". deno check clean. Co-Authored-By: Claude Opus 5 (1M context) --- .../functions/events-crawler/adapters/nvcl.ts | 36 +++++++++++-------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/supabase/functions/events-crawler/adapters/nvcl.ts b/supabase/functions/events-crawler/adapters/nvcl.ts index 71cdca8..caed9ad 100644 --- a/supabase/functions/events-crawler/adapters/nvcl.ts +++ b/supabase/functions/events-crawler/adapters/nvcl.ts @@ -239,7 +239,15 @@ export async function fetchEvents(source: Source, ctx: AdapterContext): Promise< return []; } - const candidates: Candidate[] = []; + // Deduped as the walk goes, not afterwards, because the early-stop counts against it. + // Every page repeats the same five featured rows above its paginated section, so a raw + // running total counts those once per page: with the relevance filter on — the shipped + // configuration — a source whose only relevant rows are featured ones would reach + // MAX_PER_ORG in raw count while holding a handful of distinct events, stop, and return + // that handful while in-window listings remained unread. Counting distinct links makes + // the stop condition mean what it says. + const byLink = new Map(); + let seenCandidates = 0; let pagesWalked = 0; let stoppedEarly = false; let exhausted = false; @@ -275,11 +283,17 @@ export async function fetchEvents(source: Source, ctx: AdapterContext): Promise< `extracted no title from any of them — the row markup has probably changed`, ); } - candidates.push(...parsed.candidates); + for (const c of parsed.candidates) { + seenCandidates++; + // First occurrence wins: the featured copy and the chronological copy of the same + // event carry the same date, so either is correct and keeping the earlier one is + // stable across runs. + if (!byLink.has(c.link)) byLink.set(c.link, c); + } if (parsed.lastStartMs > ctx.windowEndMs) pastWindow = true; } - if (exhausted || pastWindow || candidates.length >= MAX_PER_ORG) { + if (exhausted || pastWindow || byLink.size >= MAX_PER_ORG) { stoppedEarly = true; break; } @@ -288,26 +302,20 @@ export async function fetchEvents(source: Source, ctx: AdapterContext): Promise< if (!stoppedEarly && pagesWalked >= MAX_PAGES) { console.warn( `events-crawler: ${source.slug} stopped at the ${MAX_PAGES}-page cap with ` + - `${candidates.length} candidate(s) — listings beyond it are the furthest out and ` + + `${byLink.size} unique candidate(s) — listings beyond it are the furthest out and ` + `will be picked up on a later run.`, ); } - // Dedupe by link. Not defensive: every page repeats the same five featured rows above its - // paginated section, so without this a multi-page walk returns them once per page. - // Verified safe as an identity — across 45 sampled rows no slug appeared with two - // different dates, so the slug identifies the occurrence and matches the - // events_external_link_key unique index. - const byLink = new Map(); - for (const c of candidates) { - if (!byLink.has(c.link)) byLink.set(c.link, c); - } + // Already deduped by link during the walk (see byLink above). Link is safe as the + // identity: across 45 sampled rows no slug appeared with two different dates, so the slug + // identifies the occurrence and matches the events_external_link_key unique index. const deduped = [...byLink.values()]; deduped.sort((a, b) => a.startIso.localeCompare(b.startIso)); const selected = deduped.slice(0, MAX_PER_ORG); console.log( - `events-crawler: ${source.slug} walked ${pagesWalked} page(s), ${candidates.length} ` + + `events-crawler: ${source.slug} walked ${pagesWalked} page(s), ${seenCandidates} ` + `in-window + relevant (${deduped.length} unique), taking soonest ${selected.length}`, ); From ada14d1222c1b66dfad262caf5b1af2d79e26438 Mon Sep 17 00:00:00 2001 From: ltanafranca1004 Date: Sun, 2 Aug 2026 18:30:22 -0700 Subject: [PATCH 4/4] fix(events-crawler): detect NVCL parse failure at every extraction layer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Third CodeRabbit finding, and valid: the page-0 diagnostic only counted title misses, so a block whose anchor still matched but whose when-line or date format had moved was rejected silently. The source would return zero rows and look exactly like a library with nothing on. Closed the whole family rather than the flagged layer, because these fail independently and a per-layer fix invites the same finding again next round. The three structural rejections — no anchor or empty title, no bold when-line paragraph, when-line present but unparseable — are now counted separately, and page 0 reports the breakdown when none of its blocks survive extraction. The per-layer split is not decoration: it says which of the three moved, and the fix differs for each. The relevance and window filters are deliberately excluded from that count. They reject on merit, and rejecting most of a library calendar is precisely their job, so folding them in would make the diagnostic fire on every healthy run — this source legitimately returns zero under the filter today. Verified by breaking each layer in turn against the live feed: when-line regex broken → "title 0, when-line 15, date 0" date parse broken → "title 0, when-line 0, date 15" restored → silent, and still 0 rows from the relevance filter That last line is the one that matters: a legitimate zero stays quiet while a structural zero is loud. Happy path unchanged — filter off still gives 4 pages, 58 raw, 38 unique, soonest 25. deno check clean. Co-Authored-By: Claude Opus 5 (1M context) --- .../functions/events-crawler/adapters/nvcl.ts | 61 +++++++++++++++---- 1 file changed, 49 insertions(+), 12 deletions(-) diff --git a/supabase/functions/events-crawler/adapters/nvcl.ts b/supabase/functions/events-crawler/adapters/nvcl.ts index caed9ad..d848025 100644 --- a/supabase/functions/events-crawler/adapters/nvcl.ts +++ b/supabase/functions/events-crawler/adapters/nvcl.ts @@ -161,12 +161,32 @@ interface Candidate { endIso: string | null; } +/** Per-layer tally of blocks the parse could not read at all. */ +interface StructuralMisses { + /** No `/events/` anchor, or an anchor with an empty title. */ + title: number; + /** Anchor found, but no bold when-line paragraph in the block. */ + when: number; + /** When-line found, but it did not parse as a date (format or weekday change). */ + date: number; +} + interface PageParse { candidates: Candidate[]; /** Blocks seen, before any filtering — 0 means the listing has run out. */ blocks: number; - /** Blocks whose title link did not match, out of `blocks`. */ - titleMisses: number; + /** + * Blocks rejected for STRUCTURAL reasons — the markup or the date format moved — as + * opposed to the relevance and window filters, which reject on merit and are expected to + * reject most of a library calendar. + * + * Split by layer because each fails independently and the fix differs: the block marker + * can still match while the anchor moves, the anchor can match while the bold-paragraph + * when-line is renamed, and both can match while the date string changes shape (dropping + * the weekday would defeat the parse on its own). Counting only the first layer would + * leave the other two silent, which is the failure mode this whole set exists to prevent. + */ + misses: StructuralMisses; /** * Start of the LAST block on the page, in document order — the stop signal. * @@ -189,23 +209,32 @@ function parsePage(html: string, source: Source, ctx: AdapterContext): PageParse const chunks = html.split(BLOCK_MARKER).slice(1); const candidates: Candidate[] = []; let lastStartMs = 0; - let titleMisses = 0; + const misses: StructuralMisses = { title: 0, when: 0, date: 0 }; for (const chunk of chunks) { const titleMatch = chunk.match(TITLE_LINK_RE); if (!titleMatch) { - titleMisses++; + misses.title++; continue; } const href = titleMatch[1]; // Filter on the FULL cleaned title, then truncate for storage. const fullTitle = clean(titleMatch[2]); - if (!fullTitle) continue; + if (!fullTitle) { + misses.title++; + continue; + } const whenMatch = chunk.match(WHEN_RE); - if (!whenMatch) continue; + if (!whenMatch) { + misses.when++; + continue; + } const parsed = parseWhen(clean(whenMatch[1]), ORG_TIMEZONE, ctx.nowMs); - if (!parsed) continue; + if (!parsed) { + misses.date++; + continue; + } const startMs = Date.parse(parsed.startIso); lastStartMs = startMs; @@ -223,7 +252,7 @@ function parsePage(html: string, source: Source, ctx: AdapterContext): PageParse }); } - return { candidates, blocks: chunks.length, titleMisses, lastStartMs }; + return { candidates, blocks: chunks.length, misses, lastStartMs }; } export async function fetchEvents(source: Source, ctx: AdapterContext): Promise { @@ -275,12 +304,20 @@ export async function fetchEvents(source: Source, ctx: AdapterContext): Promise< exhausted = true; continue; } - // Blocks present but every title missed: the marker still matches while the anchor - // markup moved. Same failure mode one layer down, and just as silent without this. - if (pagesWalked === 1 && parsed.titleMisses === parsed.blocks) { + // Blocks present, but not one of them survived extraction. The marker still matches + // while something inside it moved — the anchor, the when-line, or the date format. + // Checked across all three layers rather than the title alone: each fails + // independently, and a source that returns zero rows because its date strings changed + // shape looks exactly like a library with nothing on. The relevance and window filters + // are deliberately NOT counted here — they reject on merit, and rejecting most of a + // library calendar is their job, so folding them in would make this fire constantly. + const structural = parsed.misses.title + parsed.misses.when + parsed.misses.date; + if (pagesWalked === 1 && structural === parsed.blocks) { console.error( `events-crawler: ${source.slug} matched ${parsed.blocks} block(s) on page 0 but ` + - `extracted no title from any of them — the row markup has probably changed`, + `extracted none of them (title ${parsed.misses.title}, when-line ` + + `${parsed.misses.when}, date ${parsed.misses.date}) — the row markup or the ` + + `date format has probably changed`, ); } for (const c of parsed.candidates) {