diff --git a/apps/scraper/src/scraper-contracts.ts b/apps/scraper/src/scraper-contracts.ts index bfcfd04f..da14f933 100644 --- a/apps/scraper/src/scraper-contracts.ts +++ b/apps/scraper/src/scraper-contracts.ts @@ -2,6 +2,7 @@ import type { ScraperEnvContract } from "@acme/env"; import { caSosStatementsConfig } from "./scrapers/ca-sos-statements.config.js"; import { congressConfig } from "./scrapers/congress.config.js"; +import { durhamOnBaseConfig } from "./scrapers/durham-onbase.config.js"; import { federalregisterConfig } from "./scrapers/federalregister.config.js"; import { sccCvigConfig } from "./scrapers/scc-cvig.config.js"; import { scotusConfig } from "./scrapers/scotus.config.js"; @@ -12,4 +13,5 @@ export const scraperContracts: readonly ScraperEnvContract[] = [ scotusConfig, sccCvigConfig, caSosStatementsConfig, + durhamOnBaseConfig, ]; diff --git a/apps/scraper/src/scrapers.ts b/apps/scraper/src/scrapers.ts index d4ab8d4d..dde32c89 100644 --- a/apps/scraper/src/scrapers.ts +++ b/apps/scraper/src/scrapers.ts @@ -1,6 +1,7 @@ import type { Scraper } from "./utils/types.js"; import { caSosStatements } from "./scrapers/ca-sos-statements.js"; import { congress } from "./scrapers/congress.js"; +import { durhamOnBase } from "./scrapers/durham-onbase.js"; import { federalregister } from "./scrapers/federalregister.js"; import { sccCvig } from "./scrapers/scc-cvig.js"; import { scotus } from "./scrapers/scotus.js"; @@ -11,4 +12,5 @@ export const scrapers: readonly Scraper[] = [ scotus, sccCvig, caSosStatements, + durhamOnBase, ]; diff --git a/apps/scraper/src/scrapers/__fixtures__/durham-onbase-agenda.html b/apps/scraper/src/scrapers/__fixtures__/durham-onbase-agenda.html new file mode 100644 index 00000000..55a984b8 --- /dev/null +++ b/apps/scraper/src/scrapers/__fixtures__/durham-onbase-agenda.html @@ -0,0 +1,6 @@ + +
Consent Agenda
+
1. Approval of City Council Minutes
To approve the listed minutes. [Approved by Vote: 7/0]
+
General Business Agenda
+
21. Consolidated Annexation – 4802 Cheek Road
Motion 1: To adopt the ordinance. [FAILED by Vote: 0/7] Ayes: None. Nays: Mayor Williams, Council Member Baker. Motion 2: Consider rezoning. No vote taken.
+ diff --git a/apps/scraper/src/scrapers/__fixtures__/durham-onbase-index.html b/apps/scraper/src/scrapers/__fixtures__/durham-onbase-index.html new file mode 100644 index 00000000..ed7cac0d --- /dev/null +++ b/apps/scraper/src/scrapers/__fixtures__/durham-onbase-index.html @@ -0,0 +1,3 @@ + diff --git a/apps/scraper/src/scrapers/__fixtures__/durham-onbase-item.html b/apps/scraper/src/scrapers/__fixtures__/durham-onbase-item.html new file mode 100644 index 00000000..af308ff9 --- /dev/null +++ b/apps/scraper/src/scrapers/__fixtures__/durham-onbase-item.html @@ -0,0 +1,4 @@ +

Supporting Documents

+Final-Published Attachment - Approval Memo +March 16 City Council Minutes + diff --git a/apps/scraper/src/scrapers/durham-onbase-parser.test.ts b/apps/scraper/src/scrapers/durham-onbase-parser.test.ts new file mode 100644 index 00000000..d55b0c68 --- /dev/null +++ b/apps/scraper/src/scrapers/durham-onbase-parser.test.ts @@ -0,0 +1,73 @@ +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import { describe, it } from "node:test"; + +import { + parseAgendaOutline, + parseItemAttachments, + parseMeetingIndex, +} from "./durham-onbase-parser.js"; + +const fixture = (name: string) => + readFile(new URL(`./__fixtures__/${name}`, import.meta.url), "utf8"); + +describe("Durham OnBase deterministic parsers", () => { + it("parses the embedded meeting index JSON and preserves timezone", async () => { + const meetings = parseMeetingIndex( + await fixture("durham-onbase-index.html"), + ); + assert.equal(meetings.length, 2); + assert.deepEqual( + { + id: meetings[0]?.id, + type: meetings[0]?.meetingType, + iso: meetings[0]?.date.toISOString(), + latestDocumentType: meetings[1]?.latestDocumentType, + }, + { + id: 748, + type: "City Council Meeting Agenda", + iso: "2026-05-18T23:00:00.000Z", + latestDocumentType: 2, + }, + ); + }); + + it("parses sections, items, action text, and vote text", async () => { + const items = parseAgendaOutline( + await fixture("durham-onbase-agenda.html"), + ); + assert.equal(items.length, 2); + assert.deepEqual( + { + externalId: items[0]?.externalId, + section: items[0]?.section, + number: items[0]?.agendaNumber, + title: items[0]?.title, + vote: items[0]?.voteText, + }, + { + externalId: "47768", + section: "Consent Agenda", + number: "1", + title: "Approval of City Council Minutes", + vote: "[Approved by Vote: 7/0]", + }, + ); + assert.match(items[1]?.voteText ?? "", /FAILED by Vote: 0\/7/i); + assert.match(items[1]?.voteText ?? "", /No vote taken/i); + }); + + it("parses stable attachment IDs and absolute official URLs", async () => { + const attachments = parseItemAttachments( + await fixture("durham-onbase-item.html"), + ); + assert.equal(attachments.length, 2); + assert.equal(attachments[0]?.externalId, "272813"); + assert.equal( + new URL(attachments[0]!.url).hostname, + "cityordinances.durhamnc.gov", + ); + assert.equal(attachments[1]?.title, "March 16 City Council Minutes"); + }); +}); diff --git a/apps/scraper/src/scrapers/durham-onbase-parser.ts b/apps/scraper/src/scrapers/durham-onbase-parser.ts new file mode 100644 index 00000000..0764279a --- /dev/null +++ b/apps/scraper/src/scrapers/durham-onbase-parser.ts @@ -0,0 +1,198 @@ +import * as cheerio from "cheerio"; + +export const DURHAM_ONBASE_BASE_URL = + "https://cityordinances.durhamnc.gov/OnBaseAgendaOnline/"; + +export interface OnBaseMeetingIndexItem { + id: number; + name: string; + meetingType: string; + date: Date; + location?: string; + isAgendaAvailable: boolean; + isMinutesAvailable: boolean; + agendaUniqueName: string; + minutesUniqueName: string; + latestDocumentType: 1 | 2 | 3; +} + +export interface ParsedOnBaseAttachment { + externalId: string; + title: string; + url: string; +} + +export interface ParsedOnBaseAgendaItem { + externalId: string; + section?: string; + agendaNumber?: string; + title: string; + actionText?: string; + voteText?: string; + attachments: ParsedOnBaseAttachment[]; + sortOrder: number; +} + +interface RawMeeting { + ID?: unknown; + Name?: unknown; + MeetingTypeName?: unknown; + Time?: unknown; + Location?: unknown; + IsAgendaAvailable?: unknown; + IsMinutesAvailable?: unknown; + AgendaUniqueName?: unknown; + MinutesUniqueName?: unknown; + LatestDocumentType?: unknown; +} + +function cleanText(value: string): string { + return value + .replace(/\u00a0/g, " ") + .replace(/\s+/g, " ") + .trim(); +} + +function extractJsonObject(source: string, start: number): string { + let depth = 0; + let inString = false; + let escaped = false; + + for (let index = start; index < source.length; index++) { + const char = source[index]!; + if (inString) { + if (escaped) escaped = false; + else if (char === "\\") escaped = true; + else if (char === '"') inString = false; + continue; + } + if (char === '"') inString = true; + else if (char === "{") depth++; + else if (char === "}" && --depth === 0) + return source.slice(start, index + 1); + } + + throw new Error("Unterminated OnBase meeting index JSON"); +} + +export function parseMeetingIndex(html: string): OnBaseMeetingIndexItem[] { + const marker = "showSearchResults(new SearchResults("; + const markerIndex = html.indexOf(marker); + if (markerIndex < 0) + throw new Error("OnBase meeting index payload not found"); + const jsonStart = html.indexOf("{", markerIndex + marker.length); + if (jsonStart < 0) throw new Error("OnBase meeting index JSON not found"); + + const payload = JSON.parse(extractJsonObject(html, jsonStart)) as { + Meetings?: RawMeeting[]; + }; + + return (payload.Meetings ?? []).flatMap((raw) => { + if ( + typeof raw.ID !== "number" || + typeof raw.Name !== "string" || + typeof raw.MeetingTypeName !== "string" || + typeof raw.Time !== "string" || + typeof raw.AgendaUniqueName !== "string" || + typeof raw.MinutesUniqueName !== "string" + ) { + return []; + } + const date = new Date(raw.Time); + if (Number.isNaN(date.getTime())) return []; + const latest = + raw.LatestDocumentType === 2 || raw.LatestDocumentType === 3 + ? raw.LatestDocumentType + : 1; + + return [ + { + id: raw.ID, + name: cleanText(raw.Name), + meetingType: cleanText(raw.MeetingTypeName), + date, + location: + typeof raw.Location === "string" && cleanText(raw.Location) + ? cleanText(raw.Location) + : undefined, + isAgendaAvailable: raw.IsAgendaAvailable === true, + isMinutesAvailable: raw.IsMinutesAvailable === true, + agendaUniqueName: raw.AgendaUniqueName, + minutesUniqueName: raw.MinutesUniqueName, + latestDocumentType: latest, + }, + ]; + }); +} + +function extractVoteText(actionText: string): string | undefined { + const matches = actionText.match( + /\[(?:approved|failed)[^\]]*\]|\[motion referred[^\]]*\]|no vote (?:was )?taken\.?|ayes:\s*[^.]*\.?|nays:\s*[^.]*\.?/gi, + ); + return matches ? cleanText(matches.join(" ")) : undefined; +} + +export function parseAgendaOutline(html: string): ParsedOnBaseAgendaItem[] { + const $ = cheerio.load(html); + const items: ParsedOnBaseAgendaItem[] = []; + let currentSection: string | undefined; + + $("a[href*='loadAgendaItem']").each((_, element) => { + const anchor = $(element); + const href = anchor.attr("href") ?? ""; + const match = /loadAgendaItem\((\d+),(true|false)\)/.exec(href); + if (!match) return; + const title = cleanText(anchor.text()); + if (!title) return; + if (match[2] === "true") { + currentSection = title; + return; + } + + const externalId = match[1]!; + const agendaMatch = /^(\d+[A-Za-z]?)\.\s*(.+)$/.exec(title); + const displayTitle = agendaMatch?.[2] ?? title; + const tableText = cleanText(anchor.closest("table").text()); + const actionText = cleanText( + tableText.startsWith(title) ? tableText.slice(title.length) : tableText, + ); + + items.push({ + externalId, + section: currentSection, + agendaNumber: agendaMatch?.[1], + title: displayTitle, + actionText: actionText || undefined, + voteText: actionText ? extractVoteText(actionText) : undefined, + attachments: [], + sortOrder: items.length, + }); + }); + + return items; +} + +export function parseItemAttachments( + html: string, + baseUrl = DURHAM_ONBASE_BASE_URL, +): ParsedOnBaseAttachment[] { + const $ = cheerio.load(html); + return $("a[href*='/Documents/DownloadFile/']") + .toArray() + .flatMap((element) => { + const anchor = $(element); + const href = anchor.attr("href"); + if (!href) return []; + const url = new URL(href, baseUrl); + const externalId = + url.searchParams.get("publishId") ?? + (anchor.attr("id")?.match(/(\d+)$/)?.[1] || url.toString()); + return [ + { + externalId, + title: cleanText(anchor.text()), + url: url.toString(), + }, + ]; + }); +} diff --git a/apps/scraper/src/scrapers/durham-onbase.config.ts b/apps/scraper/src/scrapers/durham-onbase.config.ts new file mode 100644 index 00000000..9297fbf7 --- /dev/null +++ b/apps/scraper/src/scrapers/durham-onbase.config.ts @@ -0,0 +1,11 @@ +import type { ScraperEnvContract } from "@acme/env"; + +export const durhamOnBaseConfig = { + id: "durham-onbase", + name: "Durham City Council OnBase", + source: "City of Durham OnBase Agenda Online", + environment: { + required: ["POSTGRES_URL"], + optional: ["DURHAM_ONBASE_MAX_ITEMS", "DURHAM_ONBASE_CACHE_TTL_HOURS"], + }, +} as const satisfies ScraperEnvContract; diff --git a/apps/scraper/src/scrapers/durham-onbase.ts b/apps/scraper/src/scrapers/durham-onbase.ts new file mode 100644 index 00000000..305db3ec --- /dev/null +++ b/apps/scraper/src/scrapers/durham-onbase.ts @@ -0,0 +1,245 @@ +import { and, eq, notInArray } from "drizzle-orm"; + +import { db } from "@acme/db/client"; +import { LocalAgendaItem, LocalMeeting } from "@acme/db/schema"; + +import type { Scraper } from "../utils/types.js"; +import { getItemLimit } from "../utils/concurrency.js"; +import { setExpectedTotal } from "../utils/db/metrics.js"; +import { fetchWithRetry } from "../utils/fetch.js"; +import { createLogger } from "../utils/log.js"; +import { + DURHAM_ONBASE_BASE_URL, + parseAgendaOutline, + parseItemAttachments, + parseMeetingIndex, +} from "./durham-onbase-parser.js"; +import { durhamOnBaseConfig } from "./durham-onbase.config.js"; + +const PROVIDER = "onbase"; +const JURISDICTION = "durham-nc"; +const MIN_REQUEST_INTERVAL_MS = 250; +const logger = createLogger(durhamOnBaseConfig.name); +let nextRequestAt = 0; +let requestGate = Promise.resolve(); + +async function fetchOnBase(path: string): Promise { + const turn = requestGate.then(async () => { + const delay = Math.max(0, nextRequestAt - Date.now()); + if (delay) await new Promise((resolve) => setTimeout(resolve, delay)); + nextRequestAt = Date.now() + MIN_REQUEST_INTERVAL_MS; + }); + requestGate = turn.catch(() => undefined); + await turn; + const response = await fetchWithRetry( + new URL(path, DURHAM_ONBASE_BASE_URL).toString(), + { + timeoutMs: 30_000, + headers: { + "User-Agent": + "Billion civic-data scraper (contact: support@billion.app)", + }, + }, + ); + return response.text(); +} + +function durhamCalendarYear(date: Date): number { + return Number( + new Intl.DateTimeFormat("en-US", { + year: "numeric", + timeZone: "America/New_York", + }).format(date), + ); +} + +function documentTypeName(documentType: 1 | 2 | 3): string { + if (documentType === 2) return "minutes"; + if (documentType === 3) return "summary"; + return "agenda"; +} + +function pdfUrl( + uniqueName: string, + documentType: number, + meetingId: number, +): string { + const path = `Documents/DownloadFile/${encodeURIComponent(uniqueName)}.pdf`; + const url = new URL(path, DURHAM_ONBASE_BASE_URL); + url.searchParams.set("documentType", String(documentType)); + url.searchParams.set("meetingId", String(meetingId)); + return url.toString(); +} + +async function scrape(maxItems = 100): Promise { + const cycleYear = new Date().getFullYear(); + const ttlHours = Number(process.env.DURHAM_ONBASE_CACHE_TTL_HOURS ?? 24); + const cacheCutoff = new Date(Date.now() - ttlHours * 60 * 60 * 1000); + logger.info(`Fetching current ${cycleYear} council cycle`); + + const indexHtml = await fetchOnBase(""); + const meetings = parseMeetingIndex(indexHtml) + .filter((meeting) => durhamCalendarYear(meeting.date) === cycleYear) + .slice(0, maxItems); + setExpectedTotal(meetings.length); + + const limit = getItemLimit(); + const results = await Promise.allSettled( + meetings.map((meeting) => + limit(async () => { + const externalId = String(meeting.id); + const [cached] = await db + .select({ fetchedAt: LocalMeeting.fetchedAt }) + .from(LocalMeeting) + .where( + and( + eq(LocalMeeting.provider, PROVIDER), + eq(LocalMeeting.jurisdiction, JURISDICTION), + eq(LocalMeeting.externalId, externalId), + ), + ) + .limit(1); + if (cached && cached.fetchedAt >= cacheCutoff) { + logger.info(`Cached: ${meeting.name}`); + return; + } + + const documentType = meeting.latestDocumentType; + const type = documentTypeName(documentType); + const outlineHtml = await fetchOnBase( + `Documents/ViewAgenda?meetingId=${meeting.id}&type=${type}&doctype=${documentType}`, + ); + const items = parseAgendaOutline(outlineHtml); + + for (const item of items) { + const detailHtml = await fetchOnBase( + `Meetings/ViewMeetingAgendaItem?meetingId=${meeting.id}&itemId=${item.externalId}&isSection=false&type=${type}`, + ); + item.attachments = parseItemAttachments(detailHtml); + } + + const sourceUrl = new URL( + `Meetings/ViewMeeting?doctype=${documentType}&id=${meeting.id}`, + DURHAM_ONBASE_BASE_URL, + ).toString(); + const [storedMeeting] = await db + .insert(LocalMeeting) + .values({ + provider: PROVIDER, + jurisdiction: JURISDICTION, + externalId, + name: meeting.name, + meetingType: meeting.meetingType, + date: meeting.date, + location: meeting.location, + sourceUrl, + agendaUrl: meeting.isAgendaAvailable + ? pdfUrl(meeting.agendaUniqueName, 1, meeting.id) + : undefined, + minutesUrl: meeting.isMinutesAvailable + ? pdfUrl(meeting.minutesUniqueName, 2, meeting.id) + : undefined, + actionAgendaUrl: sourceUrl, + fetchedAt: new Date(), + }) + .onConflictDoUpdate({ + target: [ + LocalMeeting.provider, + LocalMeeting.jurisdiction, + LocalMeeting.externalId, + ], + set: { + name: meeting.name, + meetingType: meeting.meetingType, + date: meeting.date, + location: meeting.location, + sourceUrl, + agendaUrl: meeting.isAgendaAvailable + ? pdfUrl(meeting.agendaUniqueName, 1, meeting.id) + : null, + minutesUrl: meeting.isMinutesAvailable + ? pdfUrl(meeting.minutesUniqueName, 2, meeting.id) + : null, + actionAgendaUrl: sourceUrl, + fetchedAt: new Date(), + }, + }) + .returning({ id: LocalMeeting.id }); + if (!storedMeeting) + throw new Error(`Failed to persist meeting ${externalId}`); + + for (const item of items) { + const itemSourceUrl = new URL( + `Meetings/ViewMeetingAgendaItem?meetingId=${meeting.id}&itemId=${item.externalId}&isSection=false&type=${type}`, + DURHAM_ONBASE_BASE_URL, + ).toString(); + await db + .insert(LocalAgendaItem) + .values({ + meetingId: storedMeeting.id, + externalId: item.externalId, + section: item.section, + agendaNumber: item.agendaNumber, + title: item.title, + actionText: item.actionText, + voteText: item.voteText, + attachments: item.attachments, + sourceUrl: itemSourceUrl, + sortOrder: item.sortOrder, + fetchedAt: new Date(), + }) + .onConflictDoUpdate({ + target: [LocalAgendaItem.meetingId, LocalAgendaItem.externalId], + set: { + section: item.section, + agendaNumber: item.agendaNumber, + title: item.title, + actionText: item.actionText, + voteText: item.voteText, + attachments: item.attachments, + sourceUrl: itemSourceUrl, + sortOrder: item.sortOrder, + fetchedAt: new Date(), + }, + }); + } + + if (items.length) { + await db.delete(LocalAgendaItem).where( + and( + eq(LocalAgendaItem.meetingId, storedMeeting.id), + notInArray( + LocalAgendaItem.externalId, + items.map((item) => item.externalId), + ), + ), + ); + } else { + await db + .delete(LocalAgendaItem) + .where(eq(LocalAgendaItem.meetingId, storedMeeting.id)); + } + logger.success(`Scraped ${meeting.name} (${items.length} items)`); + }), + ), + ); + + const failures = results.filter( + (result): result is PromiseRejectedResult => result.status === "rejected", + ); + if (failures.length) { + throw new AggregateError( + failures.map((failure) => failure.reason), + `${failures.length} Durham meeting(s) failed`, + ); + } + logger.success(`Completed ${meetings.length} current-cycle meetings`); +} + +export const durhamOnBase: Scraper = { + ...durhamOnBaseConfig, + scrape: (options) => + scrape( + options?.maxItems ?? Number(process.env.DURHAM_ONBASE_MAX_ITEMS ?? 100), + ), +}; diff --git a/docs/data-layer.md b/docs/data-layer.md index e6aed748..8e31728d 100644 --- a/docs/data-layer.md +++ b/docs/data-layer.md @@ -59,6 +59,14 @@ All three content tables share a common pattern: **Local government (Legistar cache)** — `legistar_body`, `legistar_matter`, `legistar_meeting`, `legistar_agenda_item`, `legistar_vote`. These cache San Jose / Santa Clara / Sunnyvale council data (ordinances, meetings, agenda items, votes) keyed by `(jurisdiction, *_id)` with a `fetched_at` timestamp. +**Provider-neutral local government** — `local_meeting` and +`local_agenda_item`. These hold systems that do not expose Legistar semantics, +starting with Durham's public OnBase Agenda Online instance. Meetings are keyed +by `(provider, jurisdiction, external_id)`; items are keyed by +`(meeting_id, external_id)` and carry structured attachment links plus official +action/vote text. The API reads persisted rows instead of scraping during a +user request. + **User engagement & caching:** | Table | Purpose | diff --git a/docs/data-sources-api.md b/docs/data-sources-api.md index fd61ad4e..a4ab420d 100644 --- a/docs/data-sources-api.md +++ b/docs/data-sources-api.md @@ -202,6 +202,25 @@ const votes = await legistar.getVotes("sanjose", matterId); To add a city: add its `*.legistar.com` subdomain to `JURISDICTIONS` in `integrations/legistar.ts`. +### Provider-neutral local meetings (Durham OnBase) + +- **Source:** [City of Durham OnBase Agenda Online](https://cityordinances.durhamnc.gov/OnBaseAgendaOnline/) +- **Reader:** `packages/api/src/integrations/local-government.ts` +- **tRPC:** `localGovernment.meetings`, `localGovernment.meeting` + +The Durham scraper persists City Council meetings and work sessions, agenda or +minutes item outlines, supporting-document URLs, and official action/vote text. +The reader is provider-neutral and serves only cached database records; it does +not contact OnBase in the request path. + +```ts +const meetings = await getLocalMeetings({ + provider: "onbase", + jurisdiction: "durham-nc", +}); +const meeting = await getLocalMeeting("onbase", "durham-nc", "748"); +``` + --- ## CA election results diff --git a/docs/scraper.md b/docs/scraper.md index 68a83d34..d424485d 100644 --- a/docs/scraper.md +++ b/docs/scraper.md @@ -25,11 +25,25 @@ process environment at runtime, not embedded during the build. | `ca-sos-statements.ts` | CA Secretary of State guide | `civic_api_cache` | official candidate-statement pages | | `ca-lao-fiscal.ts` | CA LAO ballot analyses | `civic_api_cache` | proposition fiscal analyses via HTML parse | | `ca-vig-archive.ts` | CA SOS voter-guide archive | `civic_api_cache` | historical proposition guide pages via HTML parse | +| `durham-onbase.ts` | Durham OnBase Agenda Online | `local_*` tables | meetings, items, attachments, and official actions | All HTTP goes through one `fetchWithRetry()` utility (`apps/scraper/src/utils/fetch.ts`): exponential backoff (1s/2s/4s…), `Retry-After` support (seconds or HTTP-date), 30s default timeout via `AbortController`, retriable on 429/5xx and `ECONNRESET`/`ECONNREFUSED`, plus a stateful **per-host backoff** that ramps on 429/5xx and relaxes on success. > Note: `whitehouse.gov` cheerio scraping was replaced by the structured **Federal Register** REST API. `vote411-ballot.ts` exists for address-based ballot lookup (needs Playwright) but isn't wired into the CLI. +### Durham OnBase + +Run `pnpm -F @acme/scraper start:dev -- durham-onbase`. The scraper reads the +official OnBase embedded meeting JSON, agenda/minutes HTML outlines, and agenda +item detail endpoints. It does not use AI or PDF vision. Requests are serialized +at a minimum 250 ms interval, use the shared retry/backoff client, and skip rows +refreshed in the last 24 hours by default. + +Only meetings in the current calendar-year council cycle are ingested. The +product does not expose historical election cycles or run an OnBase backfill. +`DURHAM_ONBASE_MAX_ITEMS` (default `100`) limits meetings and +`DURHAM_ONBASE_CACHE_TTL_HOURS` (default `24`) controls refresh frequency. + ## Upsert + Change Detection `apps/scraper/src/utils/db/operations.ts` centralizes writes behind a discriminated-union `upsertContent(type, data)` (`type` ∈ bill | government_content | court_case). Each run: diff --git a/packages/api/package.json b/packages/api/package.json index 661eb542..42d260d2 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -11,6 +11,10 @@ "types": "./dist/integrations/legistar.d.ts", "default": "./src/integrations/legistar.ts" }, + "./integrations/local-government": { + "types": "./dist/integrations/local-government.d.ts", + "default": "./src/integrations/local-government.ts" + }, "./lib/civic": { "types": "./dist/lib/civic.d.ts", "default": "./src/lib/civic.ts" diff --git a/packages/api/src/index.ts b/packages/api/src/index.ts index 0d729333..931cf688 100644 --- a/packages/api/src/index.ts +++ b/packages/api/src/index.ts @@ -88,6 +88,17 @@ export { LegistarError, JURISDICTIONS, } from "./integrations/legistar"; + +// Provider-neutral persisted local meetings (OnBase and future systems). +export { + getLocalMeeting, + getLocalMeetings, +} from "./integrations/local-government"; +export type { + LocalMeetingQuery, + LocalMeetingRecord, + LocalMeetingDetail, +} from "./integrations/local-government"; export type { Jurisdiction, LegistarMeeting, diff --git a/packages/api/src/integrations/local-government.ts b/packages/api/src/integrations/local-government.ts new file mode 100644 index 00000000..3914d47d --- /dev/null +++ b/packages/api/src/integrations/local-government.ts @@ -0,0 +1,60 @@ +import { and, asc, desc, eq, gte, lte } from "@acme/db"; +import { db } from "@acme/db/client"; +import { LocalAgendaItem, LocalMeeting } from "@acme/db/schema"; + +export interface LocalMeetingQuery { + provider?: string; + jurisdiction?: string; + start?: Date; + end?: Date; + limit?: number; +} + +export async function getLocalMeetings(query: LocalMeetingQuery = {}) { + const filters = []; + if (query.provider) filters.push(eq(LocalMeeting.provider, query.provider)); + if (query.jurisdiction) + filters.push(eq(LocalMeeting.jurisdiction, query.jurisdiction)); + if (query.start) filters.push(gte(LocalMeeting.date, query.start)); + if (query.end) filters.push(lte(LocalMeeting.date, query.end)); + + return db + .select() + .from(LocalMeeting) + .where(filters.length ? and(...filters) : undefined) + .orderBy(desc(LocalMeeting.date)) + .limit(Math.min(query.limit ?? 50, 100)); +} + +export async function getLocalMeeting( + provider: string, + jurisdiction: string, + externalId: string, +) { + const [meeting] = await db + .select() + .from(LocalMeeting) + .where( + and( + eq(LocalMeeting.provider, provider), + eq(LocalMeeting.jurisdiction, jurisdiction), + eq(LocalMeeting.externalId, externalId), + ), + ) + .limit(1); + if (!meeting) return null; + + const agendaItems = await db + .select() + .from(LocalAgendaItem) + .where(eq(LocalAgendaItem.meetingId, meeting.id)) + .orderBy(asc(LocalAgendaItem.sortOrder)); + return { ...meeting, agendaItems }; +} + +export type LocalMeetingRecord = Awaited< + ReturnType +>[number]; +export type LocalMeetingDetail = NonNullable< + Awaited> +>; diff --git a/packages/api/src/root.ts b/packages/api/src/root.ts index ce0f4159..c426f42d 100644 --- a/packages/api/src/root.ts +++ b/packages/api/src/root.ts @@ -3,6 +3,7 @@ import { civicRouter } from "./router/civic"; import { contentRouter } from "./router/content"; import { feedbackRouter } from "./router/feedback"; import { legistarRouter } from "./router/legistar"; +import { localGovernmentRouter } from "./router/local-government"; import { openStatesRouter } from "./router/open-states"; import { placesRouter } from "./router/places"; import { postRouter } from "./router/post"; @@ -14,6 +15,7 @@ export const appRouter = createTRPCRouter({ auth: authRouter, civic: civicRouter, legistar: legistarRouter, + localGovernment: localGovernmentRouter, openStates: openStatesRouter, places: placesRouter, post: postRouter, diff --git a/packages/api/src/router/local-government.ts b/packages/api/src/router/local-government.ts new file mode 100644 index 00000000..183c86b4 --- /dev/null +++ b/packages/api/src/router/local-government.ts @@ -0,0 +1,36 @@ +import type { TRPCRouterRecord } from "@trpc/server"; +import { z } from "zod/v4"; + +import { + getLocalMeeting, + getLocalMeetings, +} from "../integrations/local-government"; +import { publicProcedure } from "../trpc"; + +export const localGovernmentRouter = { + meetings: publicProcedure + .input( + z + .object({ + provider: z.string().min(1).optional(), + jurisdiction: z.string().min(1).optional(), + start: z.date().optional(), + end: z.date().optional(), + limit: z.number().int().min(1).max(100).default(50), + }) + .optional(), + ) + .query(({ input }) => getLocalMeetings(input)), + + meeting: publicProcedure + .input( + z.object({ + provider: z.string().min(1), + jurisdiction: z.string().min(1), + externalId: z.string().min(1), + }), + ) + .query(({ input }) => + getLocalMeeting(input.provider, input.jurisdiction, input.externalId), + ), +} satisfies TRPCRouterRecord; diff --git a/packages/db/migrations/add_provider_neutral_local_meetings.sql b/packages/db/migrations/add_provider_neutral_local_meetings.sql new file mode 100644 index 00000000..a4e50581 --- /dev/null +++ b/packages/db/migrations/add_provider_neutral_local_meetings.sql @@ -0,0 +1,44 @@ +-- Provider-neutral cache for local meeting systems that are not Legistar. +CREATE TABLE IF NOT EXISTS local_meeting ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + provider VARCHAR(50) NOT NULL, + jurisdiction VARCHAR(100) NOT NULL, + external_id VARCHAR(100) NOT NULL, + name TEXT NOT NULL, + meeting_type VARCHAR(100) NOT NULL, + date TIMESTAMPTZ NOT NULL, + location TEXT, + source_url TEXT NOT NULL, + agenda_url TEXT, + minutes_url TEXT, + action_agenda_url TEXT, + fetched_at TIMESTAMPTZ NOT NULL DEFAULT now(), + created_at TIMESTAMP NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ, + CONSTRAINT local_meeting_provider_jurisdiction_external_id_unique + UNIQUE (provider, jurisdiction, external_id) +); + +CREATE INDEX IF NOT EXISTS local_meeting_date_idx ON local_meeting (date); + +CREATE TABLE IF NOT EXISTS local_agenda_item ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + meeting_id UUID NOT NULL REFERENCES local_meeting(id) ON DELETE CASCADE, + external_id VARCHAR(100) NOT NULL, + section TEXT, + agenda_number VARCHAR(50), + title TEXT NOT NULL, + action_text TEXT, + vote_text TEXT, + attachments JSONB NOT NULL DEFAULT '[]'::jsonb, + source_url TEXT NOT NULL, + sort_order INTEGER NOT NULL, + fetched_at TIMESTAMPTZ NOT NULL DEFAULT now(), + created_at TIMESTAMP NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ, + CONSTRAINT local_agenda_item_meeting_id_external_id_unique + UNIQUE (meeting_id, external_id) +); + +CREATE INDEX IF NOT EXISTS local_agenda_item_meeting_idx + ON local_agenda_item (meeting_id); diff --git a/packages/db/src/schema.ts b/packages/db/src/schema.ts index 3c990bae..7a7616ad 100644 --- a/packages/db/src/schema.ts +++ b/packages/db/src/schema.ts @@ -645,6 +645,86 @@ export const LegistarVote = pgTable( }), ); +// Provider-neutral local government cache. Unlike the Legistar-specific tables +// above, these records can represent public meeting systems such as OnBase. +export const LocalMeeting = pgTable( + "local_meeting", + (t) => ({ + id: t.uuid().notNull().primaryKey().defaultRandom(), + provider: t.varchar({ length: 50 }).notNull(), + jurisdiction: t.varchar({ length: 100 }).notNull(), + externalId: t.varchar({ length: 100 }).notNull(), + name: t.text().notNull(), + meetingType: t.varchar({ length: 100 }).notNull(), + date: t.timestamp({ mode: "date", withTimezone: true }).notNull(), + location: t.text(), + sourceUrl: t.text().notNull(), + agendaUrl: t.text(), + minutesUrl: t.text(), + actionAgendaUrl: t.text(), + fetchedAt: t + .timestamp({ mode: "date", withTimezone: true }) + .defaultNow() + .notNull(), + createdAt: t.timestamp().defaultNow().notNull(), + updatedAt: t + .timestamp({ mode: "date", withTimezone: true }) + .$onUpdateFn(() => sql`now()`), + }), + (table) => ({ + uniqueLocalMeeting: unique().on( + table.provider, + table.jurisdiction, + table.externalId, + ), + localMeetingDateIdx: index("local_meeting_date_idx").on(table.date), + }), +); + +export interface LocalMeetingAttachment { + externalId: string; + title: string; + url: string; +} + +export const LocalAgendaItem = pgTable( + "local_agenda_item", + (t) => ({ + id: t.uuid().notNull().primaryKey().defaultRandom(), + meetingId: t + .uuid() + .notNull() + .references(() => LocalMeeting.id, { onDelete: "cascade" }), + externalId: t.varchar({ length: 100 }).notNull(), + section: t.text(), + agendaNumber: t.varchar({ length: 50 }), + title: t.text().notNull(), + actionText: t.text(), + voteText: t.text(), + attachments: t + .jsonb() + .$type() + .default([]) + .notNull(), + sourceUrl: t.text().notNull(), + sortOrder: t.integer().notNull(), + fetchedAt: t + .timestamp({ mode: "date", withTimezone: true }) + .defaultNow() + .notNull(), + createdAt: t.timestamp().defaultNow().notNull(), + updatedAt: t + .timestamp({ mode: "date", withTimezone: true }) + .$onUpdateFn(() => sql`now()`), + }), + (table) => ({ + uniqueLocalAgendaItem: unique().on(table.meetingId, table.externalId), + localAgendaMeetingIdx: index("local_agenda_item_meeting_idx").on( + table.meetingId, + ), + }), +); + // Google Civic API response cache export const CivicApiCache = pgTable( "civic_api_cache", diff --git a/packages/env/src/registry.ts b/packages/env/src/registry.ts index 6686f519..a99d8046 100644 --- a/packages/env/src/registry.ts +++ b/packages/env/src/registry.ts @@ -85,6 +85,12 @@ const scraperSourceLimitDefinitions = [ ["SCOTUS_MAX_ITEMS", "CourtListener opinion clusters per run.", "50"], ["SCC_CVIG_MAX_ITEMS", "Santa Clara voter-guide PDFs per run.", "10"], ["CA_SOS_MAX_ITEMS", "California SOS office pages per run.", "9"], + ["DURHAM_ONBASE_MAX_ITEMS", "Durham OnBase meetings per run.", "100"], + [ + "DURHAM_ONBASE_CACHE_TTL_HOURS", + "Hours before a Durham OnBase meeting is refreshed.", + "24", + ], ] as const; export const envRegistry = [