From 26d2c688e1c14308f8de2a5864b7c9851532195d Mon Sep 17 00:00:00 2001 From: ThatXliner Date: Wed, 22 Jul 2026 08:32:06 -0700 Subject: [PATCH] Implement Durham County BOCC scraper --- .../components/UpcomingMeetingsSection.tsx | 43 +- apps/scraper/README.md | 2 + apps/scraper/src/scraper-contracts.ts | 2 + apps/scraper/src/scrapers.ts | 2 + .../src/scrapers/durham-bocc.config.ts | 11 + apps/scraper/src/scrapers/durham-bocc.test.ts | 74 +++ apps/scraper/src/scrapers/durham-bocc.ts | 467 ++++++++++++++++++ .../scrapers/fixtures/durham-bocc/events.json | 78 +++ .../scrapers/fixtures/durham-bocc/items.json | 38 ++ .../scrapers/fixtures/durham-bocc/votes.json | 11 + docs/api.md | 1 + docs/data-layer.md | 2 + docs/scraper.md | 7 + packages/api/src/root.ts | 2 + packages/api/src/router/local-government.ts | 112 +++++ .../add_local_government_meetings.sql | 73 +++ packages/db/src/schema.ts | 140 ++++++ packages/env/src/registry.ts | 1 + 18 files changed, 1049 insertions(+), 17 deletions(-) create mode 100644 apps/scraper/src/scrapers/durham-bocc.config.ts create mode 100644 apps/scraper/src/scrapers/durham-bocc.test.ts create mode 100644 apps/scraper/src/scrapers/durham-bocc.ts create mode 100644 apps/scraper/src/scrapers/fixtures/durham-bocc/events.json create mode 100644 apps/scraper/src/scrapers/fixtures/durham-bocc/items.json create mode 100644 apps/scraper/src/scrapers/fixtures/durham-bocc/votes.json create mode 100644 packages/api/src/router/local-government.ts create mode 100644 packages/db/migrations/add_local_government_meetings.sql diff --git a/apps/expo/src/components/UpcomingMeetingsSection.tsx b/apps/expo/src/components/UpcomingMeetingsSection.tsx index 93bbbf27..c69ae7fb 100644 --- a/apps/expo/src/components/UpcomingMeetingsSection.tsx +++ b/apps/expo/src/components/UpcomingMeetingsSection.tsx @@ -7,16 +7,16 @@ import { import { FontAwesome } from "@expo/vector-icons"; import { useQuery } from "@tanstack/react-query"; -import type { LegistarMeeting } from "@acme/api/integrations/legistar"; +import type { RouterOutputs } from "@acme/api"; import { Text, View } from "~/components/Themed"; import { fontBody, fontEditorial, fontSize, rd, sp, useTheme } from "~/styles"; import { trpc } from "~/utils/api"; +type Meeting = RouterOutputs["localGovernment"]["listMeetings"][number]; + interface UpcomingMeetingsSectionProps { - onMeetingPress?: ( - meeting: LegistarMeeting & { jurisdiction: string }, - ) => void; + onMeetingPress?: (meeting: Meeting) => void; } function formatDate(iso: string): string { @@ -34,7 +34,7 @@ export function UpcomingMeetingsSection({ const { theme } = useTheme(); const meetingsQuery = useQuery( - trpc.legistar.getMeetings.queryOptions({ daysAhead: 30 }), + trpc.localGovernment.listMeetings.queryOptions({ daysAhead: 90 }), ); return ( @@ -47,30 +47,37 @@ export function UpcomingMeetingsSection({ {meetingsQuery.data?.slice(0, 8).map((meeting, index) => ( onMeetingPress?.(meeting)} + onPress={() => + onMeetingPress + ? onMeetingPress(meeting) + : void Linking.openURL(meeting.sourceUrl) + } activeOpacity={0.8} > {meeting.jurisdiction} - {formatDate(meeting.EventDate)} + {formatDate(meeting.startsAt.toString())} - {meeting.EventBodyName} + {meeting.isCancelled ? "Cancelled: " : ""} + {meeting.title} - {meeting.EventLocation && ( + {meeting.location && ( - {meeting.EventLocation} + {meeting.location} )} - {meeting.EventAgendaFile && ( + {meeting.documents.find((document) => document.kind === "agenda") && ( - void Linking.openURL(meeting.EventAgendaFile ?? "") + void Linking.openURL( + meeting.documents.find((document) => document.kind === "agenda")?.url ?? "", + ) } hitSlop={8} > @@ -81,10 +88,10 @@ export function UpcomingMeetingsSection({ /> )} - {meeting.EventVideoPath && ( + {meeting.videoUrl && ( - void Linking.openURL(meeting.EventVideoPath ?? "") + void Linking.openURL(meeting.videoUrl ?? "") } hitSlop={8} > @@ -95,10 +102,12 @@ export function UpcomingMeetingsSection({ /> )} - {meeting.EventMinutesFile && ( + {meeting.documents.find((document) => document.kind === "minutes") && ( - void Linking.openURL(meeting.EventMinutesFile ?? "") + void Linking.openURL( + meeting.documents.find((document) => document.kind === "minutes")?.url ?? "", + ) } hitSlop={8} > diff --git a/apps/scraper/README.md b/apps/scraper/README.md index 2e614c78..321b3726 100644 --- a/apps/scraper/README.md +++ b/apps/scraper/README.md @@ -9,6 +9,7 @@ Only these five are registered and run by `all`: | CLI name | Source and data fetched | Stored/used as | | ------------------- | ------------------------------------------------------------------------------------------ | ----------------------------------------------------------------------------------------------------- | | `federalregister` | Federal Register API presidential documents, then each document's body HTML | `government_content`; AI article/summary and feed-image enrichment | +| `durham-bocc` | Durham County's official Legistar API (current election cycle only) | Provider-neutral meetings, agenda items, actions, votes, and official document links; no AI required | | `congress` | Congress.gov API bill list, detail, CRS summaries, formatted text, and legislative actions | `bill`; powers federal bill content and AI/feed enrichment | | `scotus` | CourtListener opinion clusters, dockets, and sub-opinion text for the Supreme Court | `court_case`; powers court content and AI/feed enrichment | | `scc-cvig` | Hand-configured Santa Clara County voter-guide PDFs | Candidate statements in `CivicApiCache`; the API matches statements to candidates | @@ -124,6 +125,7 @@ CONGRESS_MAX_ITEMS=10 pnpm --filter @acme/scraper run start congress | `SCOTUS_MAX_ITEMS` | 50 | CourtListener opinion clusters | | `SCC_CVIG_MAX_ITEMS` | 10 | Voter-guide PDF documents | | `CA_SOS_MAX_ITEMS` | 9 | Statewide-office candidate-statement pages | +| `DURHAM_BOCC_MAX_ITEMS` | 100 | Current-cycle Durham County BOCC meetings | | `SCRAPER_MAX_NEW_ITEMS_PER_RUN` | 10 | New records receiving expensive AI/image enrichment | These are per-run limits, not durable calendar-day quotas. Schedule one run per diff --git a/apps/scraper/src/scraper-contracts.ts b/apps/scraper/src/scraper-contracts.ts index bfcfd04f..f6df12f3 100644 --- a/apps/scraper/src/scraper-contracts.ts +++ b/apps/scraper/src/scraper-contracts.ts @@ -3,11 +3,13 @@ import type { ScraperEnvContract } from "@acme/env"; import { caSosStatementsConfig } from "./scrapers/ca-sos-statements.config.js"; import { congressConfig } from "./scrapers/congress.config.js"; import { federalregisterConfig } from "./scrapers/federalregister.config.js"; +import { durhamBoccConfig } from "./scrapers/durham-bocc.config.js"; import { sccCvigConfig } from "./scrapers/scc-cvig.config.js"; import { scotusConfig } from "./scrapers/scotus.config.js"; export const scraperContracts: readonly ScraperEnvContract[] = [ federalregisterConfig, + durhamBoccConfig, congressConfig, scotusConfig, sccCvigConfig, diff --git a/apps/scraper/src/scrapers.ts b/apps/scraper/src/scrapers.ts index d4ab8d4d..8662cc4f 100644 --- a/apps/scraper/src/scrapers.ts +++ b/apps/scraper/src/scrapers.ts @@ -2,11 +2,13 @@ import type { Scraper } from "./utils/types.js"; import { caSosStatements } from "./scrapers/ca-sos-statements.js"; import { congress } from "./scrapers/congress.js"; import { federalregister } from "./scrapers/federalregister.js"; +import { durhamBocc } from "./scrapers/durham-bocc.js"; import { sccCvig } from "./scrapers/scc-cvig.js"; import { scotus } from "./scrapers/scotus.js"; export const scrapers: readonly Scraper[] = [ federalregister, + durhamBocc, congress, scotus, sccCvig, diff --git a/apps/scraper/src/scrapers/durham-bocc.config.ts b/apps/scraper/src/scrapers/durham-bocc.config.ts new file mode 100644 index 00000000..882a29a2 --- /dev/null +++ b/apps/scraper/src/scrapers/durham-bocc.config.ts @@ -0,0 +1,11 @@ +import type { ScraperEnvContract } from "@acme/env"; + +export const durhamBoccConfig = { + id: "durham-bocc", + name: "Durham County BOCC", + source: "Durham County Legistar Web API — BOCC meetings and actions", + environment: { + required: ["POSTGRES_URL"], + optional: ["DURHAM_BOCC_MAX_ITEMS"], + }, +} as const satisfies ScraperEnvContract; diff --git a/apps/scraper/src/scrapers/durham-bocc.test.ts b/apps/scraper/src/scrapers/durham-bocc.test.ts new file mode 100644 index 00000000..c8f66e6d --- /dev/null +++ b/apps/scraper/src/scrapers/durham-bocc.test.ts @@ -0,0 +1,74 @@ +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import test from "node:test"; + +import { + adaptDurhamItem, + adaptDurhamMeeting, + adaptDurhamVote, + currentElectionCycleStart, + parseDurhamStart, +} from "./durham-bocc.js"; + +async function fixture(name: string): Promise { + const url = new URL(`./fixtures/durham-bocc/${name}.json`, import.meta.url); + return JSON.parse(await readFile(url, "utf8")) as unknown; +} + +void test("maps regular, work, cancelled, and amended meetings", async () => { + const events = (await fixture("events")) as unknown[]; + const [regular, work, cancelled, amended] = events.map(adaptDurhamMeeting); + + assert.equal(regular?.meetingType, "Regular Session"); + assert.equal(regular?.isCancelled, false); + assert.match(regular?.videoUrl ?? "", /ID1=1539/); + assert.equal(work?.meetingType, "Work Session"); + assert.equal(cancelled?.isCancelled, true); + assert.equal(cancelled?.status, "cancelled"); + assert.equal(amended?.isAmended, true); + assert.equal(amended?.documents[0]?.title, "Amended Agenda"); +}); + +void test("maps actions, consent status, official attachments, and named votes", async () => { + const [rawItem] = (await fixture("items")) as unknown[]; + const [rawVote] = (await fixture("votes")) as unknown[]; + const item = adaptDurhamItem(rawItem); + const vote = adaptDurhamVote(rawVote); + + assert.equal(item.isConsent, true); + assert.equal(item.action, "Approved"); + assert.equal(item.outcome, "Passed"); + assert.equal(item.tally, "5-0"); + assert.equal(item.documents.length, 2); + assert.equal(item.documents[1]?.language, "es"); + assert.equal(vote.personName, "Commissioner A"); + assert.equal(vote.value, "Aye"); +}); + +void test("uses stable source ids and changes checksum when an amendment changes", async () => { + const [raw] = (await fixture("events")) as Record[]; + const original = adaptDurhamMeeting(raw); + const replaced = adaptDurhamMeeting({ + ...raw, + EventRowVersion: "replacement-v2", + EventLastModifiedUtc: "2026-01-13T00:00:00.000", + }); + + assert.equal(original.sourceId, replaced.sourceId); + assert.notEqual(original.contentHash, replaced.contentHash); +}); + +void test("parses Durham local time with DST and bounds to the current cycle", () => { + assert.equal( + parseDurhamStart("2026-01-12T00:00:00", "6:00 PM").toISOString(), + "2026-01-12T23:00:00.000Z", + ); + assert.equal( + parseDurhamStart("2026-07-13T00:00:00", "5:00 PM").toISOString(), + "2026-07-13T21:00:00.000Z", + ); + assert.equal( + currentElectionCycleStart(new Date("2026-07-21T00:00:00Z")).toISOString(), + "2025-01-01T00:00:00.000Z", + ); +}); diff --git a/apps/scraper/src/scrapers/durham-bocc.ts b/apps/scraper/src/scrapers/durham-bocc.ts new file mode 100644 index 00000000..f47a9171 --- /dev/null +++ b/apps/scraper/src/scrapers/durham-bocc.ts @@ -0,0 +1,467 @@ +import { createHash } from "node:crypto"; +import { z } from "zod/v4"; + +import type { LocalGovernmentDocument } from "@acme/db/schema"; +import { and, eq } from "@acme/db"; +import { db } from "@acme/db/client"; +import { + LocalGovernmentMeeting, + LocalGovernmentMeetingItem, + LocalGovernmentVote, +} from "@acme/db/schema"; + +import type { Scraper } from "../utils/types.js"; +import { getItemLimit } from "../utils/concurrency.js"; +import { setExpectedTotal } from "../utils/db/metrics.js"; +import { fetchWithRetry } from "../utils/fetch.js"; +import { createLogger } from "../utils/log.js"; +import { durhamBoccConfig } from "./durham-bocc.config.js"; + +const API_BASE = "https://webapi.legistar.com/v1/durhamcounty"; +const SITE_BASE = "https://durhamcounty.legistar.com"; +const PROVIDER = "legistar"; +const JURISDICTION = "Durham County, NC"; +const BOCC_BODY_ID = 138; +const SOURCE_VERSION = "durham-legistar-v1"; +const TIMEZONE = "America/New_York"; +const logger = createLogger(durhamBoccConfig.name); + +const attachmentSchema = z + .object({ + MatterAttachmentId: z.number(), + MatterAttachmentName: z.string(), + MatterAttachmentHyperlink: z.string().nullable(), + MatterAttachmentLastModifiedUtc: z.string(), + MatterAttachmentShowOnInternetPage: z.boolean().optional(), + }) + .passthrough(); + +export const durhamEventSchema = z + .object({ + EventId: z.number(), + EventGuid: z.string(), + EventLastModifiedUtc: z.string(), + EventRowVersion: z.string(), + EventBodyId: z.number(), + EventBodyName: z.string(), + EventDate: z.string(), + EventTime: z.string().nullable(), + EventAgendaStatusName: z.string().nullable(), + EventMinutesStatusName: z.string().nullable(), + EventLocation: z.string().nullable(), + EventAgendaFile: z.string().nullable(), + EventMinutesFile: z.string().nullable(), + EventComment: z.string().nullable(), + EventVideoPath: z.string().nullable(), + EventMedia: z.union([z.string(), z.number()]).nullable().optional(), + EventInSiteURL: z.string().nullable(), + }) + .passthrough(); + +export const durhamItemSchema = z + .object({ + EventItemId: z.number(), + EventItemLastModifiedUtc: z.string(), + EventItemRowVersion: z.string(), + EventItemEventId: z.number(), + EventItemAgendaSequence: z.number(), + EventItemAgendaNumber: z.string().nullable(), + EventItemAgendaNote: z.string().nullable(), + EventItemMinutesNote: z.string().nullable(), + EventItemActionName: z.string().nullable(), + EventItemActionText: z.string().nullable(), + EventItemPassedFlagName: z.string().nullable(), + EventItemRollCallFlag: z.number().nullable(), + EventItemTitle: z.string().nullable(), + EventItemTally: z.string().nullable(), + EventItemConsent: z.number(), + EventItemMover: z.string().nullable(), + EventItemSeconder: z.string().nullable(), + EventItemMatterId: z.number().nullable(), + EventItemMatterAttachments: z.array(attachmentSchema).nullable(), + }) + .passthrough(); + +export const durhamVoteSchema = z + .object({ + VoteId: z.number(), + VoteLastModifiedUtc: z.string(), + VotePersonId: z.number(), + VotePersonName: z.string(), + VoteValueName: z.string(), + VoteSort: z.number(), + VoteEventItemId: z.number(), + }) + .passthrough(); + +type DurhamEvent = z.infer; +type DurhamItem = z.infer; +type DurhamVote = z.infer; + +function hash(value: unknown): string { + return createHash("sha256").update(JSON.stringify(value)).digest("hex"); +} + +function datePart(value: string): string { + const match = /^(\d{4}-\d{2}-\d{2})/.exec(value); + if (!match) throw new Error(`Invalid Legistar date: ${value}`); + return match[1]!; +} + +function nthSunday(year: number, monthIndex: number, nth: number): number { + const first = new Date(Date.UTC(year, monthIndex, 1)).getUTCDay(); + return 1 + ((7 - first) % 7) + (nth - 1) * 7; +} + +function easternUtcOffset(date: string): "-04:00" | "-05:00" { + const [year, month, day] = date.split("-").map(Number) as [ + number, + number, + number, + ]; + const dstStart = nthSunday(year, 2, 2); + const dstEnd = nthSunday(year, 10, 1); + const isDst = + (month > 3 && month < 11) || + (month === 3 && day >= dstStart) || + (month === 11 && day < dstEnd); + return isDst ? "-04:00" : "-05:00"; +} + +export function parseDurhamStart( + eventDate: string, + eventTime: string | null, +): Date { + const date = datePart(eventDate); + const match = eventTime?.trim().match(/^(\d{1,2}):(\d{2})\s*([AP]M)$/i); + let hour = 0; + let minute = 0; + if (match) { + hour = Number(match[1]) % 12; + minute = Number(match[2]); + if (match[3]!.toUpperCase() === "PM") hour += 12; + } + return new Date( + `${date}T${String(hour).padStart(2, "0")}:${String(minute).padStart(2, "0")}:00${easternUtcOffset(date)}`, + ); +} + +export function currentElectionCycleStart(now = new Date()): Date { + const year = now.getUTCFullYear(); + const startYear = year % 2 === 0 ? year - 1 : year; + return new Date(Date.UTC(startYear, 0, 1)); +} + +function documentLanguage(title: string): string | undefined { + return /\b(spanish|español|espanol)\b/i.test(title) ? "es" : undefined; +} + +function visibleAttachmentDocuments( + item: DurhamItem, +): LocalGovernmentDocument[] { + const kind = /\bminutes?\b/i.test(item.EventItemTitle ?? "") + ? ("minutes" as const) + : ("attachment" as const); + return (item.EventItemMatterAttachments ?? []) + .filter( + (attachment) => + attachment.MatterAttachmentHyperlink && + attachment.MatterAttachmentShowOnInternetPage !== false, + ) + .map((attachment) => ({ + kind, + title: attachment.MatterAttachmentName, + url: attachment.MatterAttachmentHyperlink!, + ...(documentLanguage(attachment.MatterAttachmentName) + ? { language: documentLanguage(attachment.MatterAttachmentName) } + : {}), + })); +} + +export function adaptDurhamMeeting(input: unknown) { + const event = durhamEventSchema.parse(input); + const comment = event.EventComment?.trim() || "Board meeting"; + const markerText = [ + comment, + event.EventAgendaStatusName, + event.EventMinutesStatusName, + event.EventAgendaFile, + event.EventMinutesFile, + ] + .filter(Boolean) + .join(" "); + const isCancelled = /\bcancel(?:led|ed|lation)?\b/i.test(markerText); + const isAmended = /\b(amend(?:ed|ment)?|revis(?:ed|ion)|corrected)\b/i.test( + markerText, + ); + const documents: LocalGovernmentDocument[] = []; + if (event.EventAgendaFile) { + documents.push({ + kind: "agenda", + title: `${isAmended ? "Amended " : ""}Agenda`, + url: event.EventAgendaFile, + }); + } + if (event.EventMinutesFile) { + documents.push({ + kind: "minutes", + title: "Minutes", + url: event.EventMinutesFile, + }); + } + const sourceUrl = + event.EventInSiteURL ?? + `${SITE_BASE}/MeetingDetail.aspx?LEGID=${event.EventId}`; + const videoUrl = event.EventVideoPath + ? event.EventVideoPath + : event.EventMedia + ? `${SITE_BASE}/Video.aspx?Mode=Granicus&ID1=${event.EventMedia}&Mode2=Video` + : null; + const status = isCancelled + ? "cancelled" + : (event.EventMinutesStatusName ?? + event.EventAgendaStatusName ?? + "scheduled"); + const mapped = { + provider: PROVIDER, + jurisdiction: JURISDICTION, + sourceId: String(event.EventId), + bodyName: event.EventBodyName, + title: comment, + meetingType: comment.replace(/^cancel(?:led|ed)\s*-?\s*/i, ""), + startsAt: parseDurhamStart(event.EventDate, event.EventTime), + timezone: TIMEZONE, + location: event.EventLocation, + status, + isCancelled, + isAmended, + sourceUrl, + videoUrl, + documents, + sourceVersion: `${SOURCE_VERSION}:${event.EventRowVersion}`, + sourceUpdatedAt: new Date(event.EventLastModifiedUtc), + }; + return { ...mapped, contentHash: hash(mapped) }; +} + +export function adaptDurhamItem(input: unknown) { + const item = durhamItemSchema.parse(input); + const mapped = { + provider: PROVIDER, + jurisdiction: JURISDICTION, + sourceId: String(item.EventItemId), + meetingSourceId: String(item.EventItemEventId), + sequence: item.EventItemAgendaSequence, + agendaNumber: item.EventItemAgendaNumber, + title: item.EventItemTitle?.trim() || "Untitled agenda item", + agendaNote: item.EventItemAgendaNote, + minutesNote: item.EventItemMinutesNote, + isConsent: item.EventItemConsent === 1, + action: item.EventItemActionName, + actionText: item.EventItemActionText, + outcome: item.EventItemPassedFlagName, + tally: item.EventItemTally, + mover: item.EventItemMover, + seconder: item.EventItemSeconder, + documents: visibleAttachmentDocuments(item), + sourceVersion: `${SOURCE_VERSION}:${item.EventItemRowVersion}`, + sourceUpdatedAt: new Date(item.EventItemLastModifiedUtc), + }; + return { ...mapped, contentHash: hash(mapped) }; +} + +export function adaptDurhamVote(input: unknown) { + const vote = durhamVoteSchema.parse(input); + return { + provider: PROVIDER, + jurisdiction: JURISDICTION, + sourceId: String(vote.VoteId), + itemSourceId: String(vote.VoteEventItemId), + personSourceId: String(vote.VotePersonId), + personName: vote.VotePersonName, + value: vote.VoteValueName, + sort: vote.VoteSort, + sourceUpdatedAt: new Date(vote.VoteLastModifiedUtc), + }; +} + +async function fetchJson(url: URL): Promise { + const response = await fetchWithRetry(url.toString(), { timeoutMs: 30_000 }); + return response.json() as Promise; +} + +async function upsertMeeting(meeting: ReturnType) { + const fetchedAt = new Date(); + await db + .insert(LocalGovernmentMeeting) + .values({ ...meeting, fetchedAt }) + .onConflictDoUpdate({ + target: [ + LocalGovernmentMeeting.provider, + LocalGovernmentMeeting.jurisdiction, + LocalGovernmentMeeting.sourceId, + ], + set: { ...meeting, fetchedAt }, + }); +} + +async function upsertItem(item: ReturnType) { + const fetchedAt = new Date(); + await db + .insert(LocalGovernmentMeetingItem) + .values({ ...item, fetchedAt }) + .onConflictDoUpdate({ + target: [ + LocalGovernmentMeetingItem.provider, + LocalGovernmentMeetingItem.jurisdiction, + LocalGovernmentMeetingItem.sourceId, + ], + set: { ...item, fetchedAt }, + }); +} + +async function upsertVote(vote: ReturnType) { + const fetchedAt = new Date(); + await db + .insert(LocalGovernmentVote) + .values({ ...vote, fetchedAt }) + .onConflictDoUpdate({ + target: [ + LocalGovernmentVote.provider, + LocalGovernmentVote.jurisdiction, + LocalGovernmentVote.sourceId, + ], + set: { ...vote, fetchedAt }, + }); +} + +async function scrapeMeeting(rawEvent: unknown): Promise { + const meeting = adaptDurhamMeeting(rawEvent); + await upsertMeeting(meeting); + + const itemsUrl = new URL(`${API_BASE}/Events/${meeting.sourceId}/EventItems`); + itemsUrl.searchParams.set("AgendaNote", "1"); + itemsUrl.searchParams.set("MinutesNote", "1"); + itemsUrl.searchParams.set("Attachments", "1"); + const rawItems = z.array(z.unknown()).parse(await fetchJson(itemsUrl)); + + for (const rawItem of rawItems) { + try { + const parsed = durhamItemSchema.parse(rawItem); + let enrichedItem: unknown = rawItem; + if ( + parsed.EventItemMatterId && + (parsed.EventItemMatterAttachments?.length ?? 0) === 0 + ) { + try { + const attachmentsUrl = new URL( + `${API_BASE}/Matters/${parsed.EventItemMatterId}/Attachments`, + ); + const attachments = z + .array(attachmentSchema) + .parse(await fetchJson(attachmentsUrl)); + enrichedItem = { + ...parsed, + EventItemMatterAttachments: attachments, + }; + } catch (error) { + logger.warn( + `Attachments unavailable for matter ${parsed.EventItemMatterId}`, + error, + ); + } + } + const item = adaptDurhamItem(enrichedItem); + await upsertItem(item); + if (parsed.EventItemRollCallFlag === 1) { + const votesUrl = new URL( + `${API_BASE}/EventItems/${item.sourceId}/Votes`, + ); + const rawVotes = z.array(z.unknown()).parse(await fetchJson(votesUrl)); + for (const rawVote of rawVotes) { + try { + await upsertVote(adaptDurhamVote(rawVote)); + } catch (error) { + logger.warn( + `Skipping invalid vote for item ${item.sourceId}`, + error, + ); + } + } + } + } catch (error) { + logger.warn( + `Skipping invalid item for meeting ${meeting.sourceId}`, + error, + ); + } + } + + logger.success(`Synced ${meeting.title} (${meeting.sourceId})`); +} + +async function scrape(maxItems = 100): Promise { + const start = currentElectionCycleStart(); + const eventsUrl = new URL(`${API_BASE}/Events`); + eventsUrl.searchParams.set( + "$filter", + `EventDate ge datetime'${start.toISOString().slice(0, 10)}' and EventBodyId eq ${BOCC_BODY_ID}`, + ); + eventsUrl.searchParams.set("$orderby", "EventDate asc"); + eventsUrl.searchParams.set("$top", String(maxItems)); + + logger.info( + `Syncing current election cycle from ${start.toISOString().slice(0, 10)} (structured source; OCR not required)`, + ); + const rawEvents = z + .array(z.unknown()) + .parse(await fetchJson(eventsUrl)) + .slice(0, maxItems); + setExpectedTotal(rawEvents.length); + + const limit = getItemLimit(); + const results = await Promise.allSettled( + rawEvents.map((rawEvent) => + limit(async () => { + try { + await scrapeMeeting(rawEvent); + } catch (error) { + let sourceId = "unknown"; + const parsed = durhamEventSchema.safeParse(rawEvent); + if (parsed.success) sourceId = String(parsed.data.EventId); + logger.error( + `Meeting ${sourceId} failed without aborting run`, + error, + ); + } + }), + ), + ); + const failed = results.filter((result) => result.status === "rejected"); + if (failed.length > 0) logger.warn(`${failed.length} meeting tasks failed`); + logger.success(`Completed ${rawEvents.length} Durham BOCC meetings`); +} + +export const durhamBocc: Scraper = { + ...durhamBoccConfig, + scrape: (options) => + scrape( + (options?.maxItems ?? Number(process.env.DURHAM_BOCC_MAX_ITEMS)) || 100, + ), +}; + +// Exported for targeted cleanup/diagnostics without widening the public API. +export async function hasDurhamMeeting(sourceId: string): Promise { + const rows = await db + .select({ id: LocalGovernmentMeeting.id }) + .from(LocalGovernmentMeeting) + .where( + and( + eq(LocalGovernmentMeeting.provider, PROVIDER), + eq(LocalGovernmentMeeting.jurisdiction, JURISDICTION), + eq(LocalGovernmentMeeting.sourceId, sourceId), + ), + ) + .limit(1); + return rows.length > 0; +} diff --git a/apps/scraper/src/scrapers/fixtures/durham-bocc/events.json b/apps/scraper/src/scrapers/fixtures/durham-bocc/events.json new file mode 100644 index 00000000..fe275d86 --- /dev/null +++ b/apps/scraper/src/scrapers/fixtures/durham-bocc/events.json @@ -0,0 +1,78 @@ +[ + { + "EventId": 1439, + "EventGuid": "4AF37432-715B-4E88-8C23-C24F0ABF3D98", + "EventLastModifiedUtc": "2026-01-12T20:19:16.093", + "EventRowVersion": "AAAAAACFTNc=", + "EventBodyId": 138, + "EventBodyName": "Board of County Commissioners", + "EventDate": "2026-01-12T00:00:00", + "EventTime": "6:00 PM", + "EventAgendaStatusName": "Final", + "EventMinutesStatusName": "Draft", + "EventLocation": "Commissioners' Chambers", + "EventAgendaFile": "https://durhamcounty.legistar1.com/durhamcounty/meetings/2026/1/1439_Agenda.pdf", + "EventMinutesFile": null, + "EventComment": "Regular Session", + "EventVideoPath": null, + "EventMedia": "1539", + "EventInSiteURL": "https://durhamcounty.legistar.com/MeetingDetail.aspx?LEGID=1439" + }, + { + "EventId": 1441, + "EventGuid": "AA819B77-C3F4-49A5-8BF0-29BCDDD09B25", + "EventLastModifiedUtc": "2026-02-02T12:55:38.687", + "EventRowVersion": "AAAAAACFhUA=", + "EventBodyId": 138, + "EventBodyName": "Board of County Commissioners", + "EventDate": "2026-02-02T00:00:00", + "EventTime": "9:00 AM", + "EventAgendaStatusName": "Final", + "EventMinutesStatusName": "Draft", + "EventLocation": "Commissioners' Chambers", + "EventAgendaFile": "https://durhamcounty.legistar1.com/durhamcounty/meetings/2026/2/1441_Agenda.pdf", + "EventMinutesFile": null, + "EventComment": "Work Session", + "EventVideoPath": null, + "EventMedia": null, + "EventInSiteURL": "https://durhamcounty.legistar.com/MeetingDetail.aspx?LEGID=1441" + }, + { + "EventId": 1440, + "EventGuid": "418F96F6-84B8-40B6-9BD8-F409555EDFD7", + "EventLastModifiedUtc": "2026-01-26T01:33:13.067", + "EventRowVersion": "AAAAAACFIbU=", + "EventBodyId": 138, + "EventBodyName": "Board of County Commissioners", + "EventDate": "2026-01-26T00:00:00", + "EventTime": "7:00 PM", + "EventAgendaStatusName": "Final", + "EventMinutesStatusName": "Draft", + "EventLocation": "Commissioners' Chambers", + "EventAgendaFile": "https://durhamcounty.legistar1.com/durhamcounty/meetings/2026/1/1440_Agenda.pdf", + "EventMinutesFile": null, + "EventComment": "Cancelled - Regular Session", + "EventVideoPath": null, + "EventMedia": null, + "EventInSiteURL": "https://durhamcounty.legistar.com/MeetingDetail.aspx?LEGID=1440" + }, + { + "EventId": 1499, + "EventGuid": "EDC0142F-3923-4A79-AAA7-9319E811C112", + "EventLastModifiedUtc": "2026-05-04T15:00:00.000", + "EventRowVersion": "AMENDED-V2", + "EventBodyId": 138, + "EventBodyName": "Board of County Commissioners", + "EventDate": "2026-05-04T00:00:00", + "EventTime": "9:00 AM", + "EventAgendaStatusName": "Final", + "EventMinutesStatusName": "Draft", + "EventLocation": "Commissioners' Chambers", + "EventAgendaFile": "https://durhamcounty.legistar1.com/durhamcounty/meetings/2026/5/1499_Amended_Agenda.pdf", + "EventMinutesFile": null, + "EventComment": "Amended Work Session", + "EventVideoPath": null, + "EventMedia": "1599", + "EventInSiteURL": "https://durhamcounty.legistar.com/MeetingDetail.aspx?LEGID=1499" + } +] diff --git a/apps/scraper/src/scrapers/fixtures/durham-bocc/items.json b/apps/scraper/src/scrapers/fixtures/durham-bocc/items.json new file mode 100644 index 00000000..cfc2a009 --- /dev/null +++ b/apps/scraper/src/scrapers/fixtures/durham-bocc/items.json @@ -0,0 +1,38 @@ +[ + { + "EventItemId": 30280, + "EventItemLastModifiedUtc": "2026-04-14T13:52:55.327", + "EventItemRowVersion": "ITEM-V1", + "EventItemEventId": 1439, + "EventItemAgendaSequence": 12, + "EventItemAgendaNumber": "6.", + "EventItemAgendaNote": "Approve the consent agenda.", + "EventItemMinutesNote": "The motion carried unanimously.", + "EventItemActionName": "Approved", + "EventItemActionText": "Adopt the consent agenda", + "EventItemPassedFlagName": "Passed", + "EventItemRollCallFlag": 1, + "EventItemTitle": "Consent Agenda", + "EventItemTally": "5-0", + "EventItemConsent": 1, + "EventItemMover": "Commissioner A", + "EventItemSeconder": "Commissioner B", + "EventItemMatterId": 9181, + "EventItemMatterAttachments": [ + { + "MatterAttachmentId": 22223, + "MatterAttachmentName": "Budget memorandum", + "MatterAttachmentHyperlink": "https://durhamcounty.legistar1.com/durhamcounty/attachments/budget.pdf", + "MatterAttachmentLastModifiedUtc": "2026-04-09T14:39:50.21", + "MatterAttachmentShowOnInternetPage": true + }, + { + "MatterAttachmentId": 22224, + "MatterAttachmentName": "Presupuesto - Español", + "MatterAttachmentHyperlink": "https://durhamcounty.legistar1.com/durhamcounty/attachments/presupuesto.pdf", + "MatterAttachmentLastModifiedUtc": "2026-04-09T14:40:50.21", + "MatterAttachmentShowOnInternetPage": true + } + ] + } +] diff --git a/apps/scraper/src/scrapers/fixtures/durham-bocc/votes.json b/apps/scraper/src/scrapers/fixtures/durham-bocc/votes.json new file mode 100644 index 00000000..55e7c2c7 --- /dev/null +++ b/apps/scraper/src/scrapers/fixtures/durham-bocc/votes.json @@ -0,0 +1,11 @@ +[ + { + "VoteId": 9001, + "VoteLastModifiedUtc": "2026-04-14T14:00:00.000", + "VotePersonId": 101, + "VotePersonName": "Commissioner A", + "VoteValueName": "Aye", + "VoteSort": 1, + "VoteEventItemId": 30280 + } +] diff --git a/docs/api.md b/docs/api.md index 2f6381a2..8ac7adc0 100644 --- a/docs/api.md +++ b/docs/api.md @@ -32,6 +32,7 @@ The `civic` router calls the **Google Civic Information API** (`GOOGLE_CIVIC_API Other live civic integrations: - **`legistar`** — scrapes Legistar instances for San Jose, Santa Clara County, and Sunnyvale (no key required). +- **`localGovernment`** — reads persisted provider-neutral meetings. `listMeetings` returns bounded meeting summaries and official links; `getMeeting` returns agenda items, actions, attachments, and named votes without live source calls. - **`openStates`** — California bills, legislators, and votes via the Open States v3 API (`OPEN_STATES_API_KEY`). - **`places`** — Google **Places Autocomplete (New)** for the ballot address entry (`packages/api/src/lib/places.ts`). `autocomplete` returns US street-address predictions (biased `includedRegionCodes: ["us"]`, `includedPrimaryTypes: street_address/premise/subpremise`) for queries ≥3 chars; `details` resolves a `placeId` to its full `formattedAddress` (the ZIP the prediction omits, which Civic wants). A **session token** (UUID stable across one address entry) bundles all keystroke calls plus the closing `details` into a single billed unit. Reuses `GOOGLE_PLACES_API_KEY` → `GOOGLE_API_KEY` → `GOOGLE_CIVIC_API_KEY`; with no key it serves a small mock list so the dropdown still works in dev (same fallback pattern as `civic`). diff --git a/docs/data-layer.md b/docs/data-layer.md index e6aed748..9204c84c 100644 --- a/docs/data-layer.md +++ b/docs/data-layer.md @@ -59,6 +59,8 @@ All three content tables share a common pattern: **Local government (Legistar cache)** — `legistar_body`, `legistar_matter`, `legistar_meeting`, `legistar_agenda_item`, `legistar_vote`. These cache San Jose / Santa Clara / Sunnyvale council data (ordinances, meetings, agenda items, votes) keyed by `(jurisdiction, *_id)` with a `fetched_at` timestamp. +**Provider-neutral local government** — `local_government_meeting`, `local_government_meeting_item`, and `local_government_vote`. Source adapters retain `(provider, jurisdiction, source_id)` while readers receive one shape containing cancellation/amendment state, official documents, actions, outcomes, and named votes. Durham County BOCC is the first writer. Apply `packages/db/migrations/add_local_government_meetings.sql` before running that scraper. + **User engagement & caching:** | Table | Purpose | diff --git a/docs/scraper.md b/docs/scraper.md index 68a83d34..d9680d69 100644 --- a/docs/scraper.md +++ b/docs/scraper.md @@ -25,11 +25,18 @@ process environment at runtime, not embedded during the build. | `ca-sos-statements.ts` | CA Secretary of State guide | `civic_api_cache` | official candidate-statement pages | | `ca-lao-fiscal.ts` | CA LAO ballot analyses | `civic_api_cache` | proposition fiscal analyses via HTML parse | | `ca-vig-archive.ts` | CA SOS voter-guide archive | `civic_api_cache` | historical proposition guide pages via HTML parse | +| `durham-bocc.ts` | Durham County Legistar API | local government | current-cycle meetings, items, actions, votes, documents | All HTTP goes through one `fetchWithRetry()` utility (`apps/scraper/src/utils/fetch.ts`): exponential backoff (1s/2s/4s…), `Retry-After` support (seconds or HTTP-date), 30s default timeout via `AbortController`, retriable on 429/5xx and `ECONNRESET`/`ECONNREFUSED`, plus a stateful **per-host backoff** that ramps on 429/5xx and relaxes on success. > Note: `whitehouse.gov` cheerio scraping was replaced by the structured **Federal Register** REST API. `vote411-ballot.ts` exists for address-based ballot lookup (needs Playwright) but isn't wired into the CLI. +### Durham County BOCC + +`durham-bocc` reads Durham County's official structured Legistar feed for BOCC body ID `138`. It bounds discovery to the current two-year election cycle, upserts stable provider IDs, and stores checksums/source row versions so replaced agendas update the existing meeting. Cancellations and explicit amendments remain visible; Spanish attachments are tagged separately. Agenda/minutes PDFs are retained as official links and are not sent through AI or OCR when structured item/action fields are available. + +Run it with `pnpm --filter @acme/scraper run start durham-bocc`. The default cap is 100 meetings and can be changed with `DURHAM_BOCC_MAX_ITEMS` or `--max-items`. + ## Upsert + Change Detection `apps/scraper/src/utils/db/operations.ts` centralizes writes behind a discriminated-union `upsertContent(type, data)` (`type` ∈ bill | government_content | court_case). Each run: diff --git a/packages/api/src/root.ts b/packages/api/src/root.ts index ce0f4159..c426f42d 100644 --- a/packages/api/src/root.ts +++ b/packages/api/src/root.ts @@ -3,6 +3,7 @@ import { civicRouter } from "./router/civic"; import { contentRouter } from "./router/content"; import { feedbackRouter } from "./router/feedback"; import { legistarRouter } from "./router/legistar"; +import { localGovernmentRouter } from "./router/local-government"; import { openStatesRouter } from "./router/open-states"; import { placesRouter } from "./router/places"; import { postRouter } from "./router/post"; @@ -14,6 +15,7 @@ export const appRouter = createTRPCRouter({ auth: authRouter, civic: civicRouter, legistar: legistarRouter, + localGovernment: localGovernmentRouter, openStates: openStatesRouter, places: placesRouter, post: postRouter, diff --git a/packages/api/src/router/local-government.ts b/packages/api/src/router/local-government.ts new file mode 100644 index 00000000..175dd0ee --- /dev/null +++ b/packages/api/src/router/local-government.ts @@ -0,0 +1,112 @@ +import type { TRPCRouterRecord } from "@trpc/server"; +import { TRPCError } from "@trpc/server"; +import { and, asc, eq, gte, inArray, lte } from "@acme/db"; +import { db } from "@acme/db/client"; +import { + LocalGovernmentMeeting, + LocalGovernmentMeetingItem, + LocalGovernmentVote, +} from "@acme/db/schema"; +import { z } from "zod/v4"; + +import { publicProcedure } from "../trpc"; + +export const localGovernmentRouter = { + listMeetings: publicProcedure + .input( + z + .object({ + provider: z.string().min(1).optional(), + jurisdiction: z.string().min(1).optional(), + daysAhead: z.number().int().min(1).max(180).default(90), + includePastDays: z.number().int().min(0).max(90).default(0), + }) + .optional(), + ) + .query(async ({ input }) => { + const start = new Date(); + start.setDate(start.getDate() - (input?.includePastDays ?? 0)); + const end = new Date(); + end.setDate(end.getDate() + (input?.daysAhead ?? 90)); + const filters = [ + gte(LocalGovernmentMeeting.startsAt, start), + lte(LocalGovernmentMeeting.startsAt, end), + ]; + if (input?.provider) { + filters.push(eq(LocalGovernmentMeeting.provider, input.provider)); + } + if (input?.jurisdiction) { + filters.push( + eq(LocalGovernmentMeeting.jurisdiction, input.jurisdiction), + ); + } + return db + .select() + .from(LocalGovernmentMeeting) + .where(and(...filters)) + .orderBy(asc(LocalGovernmentMeeting.startsAt)); + }), + + getMeeting: publicProcedure + .input( + z.object({ + provider: z.string().min(1), + jurisdiction: z.string().min(1), + sourceId: z.string().min(1), + }), + ) + .query(async ({ input }) => { + const [meeting] = await db + .select() + .from(LocalGovernmentMeeting) + .where( + and( + eq(LocalGovernmentMeeting.provider, input.provider), + eq(LocalGovernmentMeeting.jurisdiction, input.jurisdiction), + eq(LocalGovernmentMeeting.sourceId, input.sourceId), + ), + ) + .limit(1); + if (!meeting) { + throw new TRPCError({ code: "NOT_FOUND", message: "Meeting not found" }); + } + + const items = await db + .select() + .from(LocalGovernmentMeetingItem) + .where( + and( + eq(LocalGovernmentMeetingItem.provider, input.provider), + eq(LocalGovernmentMeetingItem.jurisdiction, input.jurisdiction), + eq(LocalGovernmentMeetingItem.meetingSourceId, input.sourceId), + ), + ) + .orderBy(asc(LocalGovernmentMeetingItem.sequence)); + const itemIds = items.map((item) => item.sourceId); + const votes = + itemIds.length === 0 + ? [] + : await db + .select() + .from(LocalGovernmentVote) + .where( + and( + eq(LocalGovernmentVote.provider, input.provider), + eq(LocalGovernmentVote.jurisdiction, input.jurisdiction), + inArray(LocalGovernmentVote.itemSourceId, itemIds), + ), + ) + .orderBy( + asc(LocalGovernmentVote.itemSourceId), + asc(LocalGovernmentVote.sort), + ); + + return { + ...meeting, + items: items.map((item) => ({ + ...item, + votes: votes.filter((vote) => vote.itemSourceId === item.sourceId), + })), + }; + }), +} satisfies TRPCRouterRecord; diff --git a/packages/db/migrations/add_local_government_meetings.sql b/packages/db/migrations/add_local_government_meetings.sql new file mode 100644 index 00000000..16da6b12 --- /dev/null +++ b/packages/db/migrations/add_local_government_meetings.sql @@ -0,0 +1,73 @@ +CREATE TABLE IF NOT EXISTS "local_government_meeting" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL, + "provider" varchar(50) NOT NULL, + "jurisdiction" varchar(100) NOT NULL, + "source_id" varchar(100) NOT NULL, + "body_name" varchar(256) NOT NULL, + "title" text NOT NULL, + "meeting_type" varchar(100), + "starts_at" timestamp with time zone NOT NULL, + "timezone" varchar(64) NOT NULL, + "location" text, + "status" varchar(100) NOT NULL, + "is_cancelled" boolean DEFAULT false NOT NULL, + "is_amended" boolean DEFAULT false NOT NULL, + "source_url" text NOT NULL, + "video_url" text, + "documents" jsonb DEFAULT '[]'::jsonb NOT NULL, + "source_version" varchar(100) NOT NULL, + "content_hash" varchar(64) NOT NULL, + "source_updated_at" timestamp with time zone NOT NULL, + "fetched_at" timestamp with time zone DEFAULT now() NOT NULL, + "created_at" timestamp DEFAULT now() NOT NULL, + "updated_at" timestamp, + CONSTRAINT "local_government_meeting_provider_jurisdiction_source_id_unique" UNIQUE("provider", "jurisdiction", "source_id") +); +CREATE INDEX IF NOT EXISTS "local_government_meeting_starts_at_idx" ON "local_government_meeting" ("starts_at"); + +CREATE TABLE IF NOT EXISTS "local_government_meeting_item" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL, + "provider" varchar(50) NOT NULL, + "jurisdiction" varchar(100) NOT NULL, + "source_id" varchar(100) NOT NULL, + "meeting_source_id" varchar(100) NOT NULL, + "sequence" integer NOT NULL, + "agenda_number" varchar(50), + "title" text NOT NULL, + "agenda_note" text, + "minutes_note" text, + "is_consent" boolean DEFAULT false NOT NULL, + "action" text, + "action_text" text, + "outcome" varchar(100), + "tally" varchar(100), + "mover" varchar(256), + "seconder" varchar(256), + "documents" jsonb DEFAULT '[]'::jsonb NOT NULL, + "source_version" varchar(100) NOT NULL, + "content_hash" varchar(64) NOT NULL, + "source_updated_at" timestamp with time zone NOT NULL, + "fetched_at" timestamp with time zone DEFAULT now() NOT NULL, + "created_at" timestamp DEFAULT now() NOT NULL, + "updated_at" timestamp, + CONSTRAINT "local_government_meeting_item_provider_jurisdiction_source_id_unique" UNIQUE("provider", "jurisdiction", "source_id") +); +CREATE INDEX IF NOT EXISTS "local_government_item_meeting_source_idx" ON "local_government_meeting_item" ("provider", "jurisdiction", "meeting_source_id"); + +CREATE TABLE IF NOT EXISTS "local_government_vote" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL, + "provider" varchar(50) NOT NULL, + "jurisdiction" varchar(100) NOT NULL, + "source_id" varchar(100) NOT NULL, + "item_source_id" varchar(100) NOT NULL, + "person_source_id" varchar(100), + "person_name" varchar(256) NOT NULL, + "value" varchar(100) NOT NULL, + "sort" integer DEFAULT 0 NOT NULL, + "source_updated_at" timestamp with time zone NOT NULL, + "fetched_at" timestamp with time zone DEFAULT now() NOT NULL, + "created_at" timestamp DEFAULT now() NOT NULL, + "updated_at" timestamp, + CONSTRAINT "local_government_vote_provider_jurisdiction_source_id_unique" UNIQUE("provider", "jurisdiction", "source_id") +); +CREATE INDEX IF NOT EXISTS "local_government_vote_item_source_idx" ON "local_government_vote" ("provider", "jurisdiction", "item_source_id"); diff --git a/packages/db/src/schema.ts b/packages/db/src/schema.ts index 3c990bae..b7710b15 100644 --- a/packages/db/src/schema.ts +++ b/packages/db/src/schema.ts @@ -645,6 +645,146 @@ export const LegistarVote = pgTable( }), ); +// Provider-neutral local government meetings. Source adapters (Legistar, +// CivicPlus, etc.) retain their native IDs while readers use one stable shape. +export interface LocalGovernmentDocument { + kind: "agenda" | "minutes" | "packet" | "attachment" | "notice" | "other"; + title: string; + url: string; + language?: string; +} + +export const LocalGovernmentMeeting = pgTable( + "local_government_meeting", + (t) => ({ + id: t.uuid().notNull().primaryKey().defaultRandom(), + provider: t.varchar({ length: 50 }).notNull(), + jurisdiction: t.varchar({ length: 100 }).notNull(), + sourceId: t.varchar({ length: 100 }).notNull(), + bodyName: t.varchar({ length: 256 }).notNull(), + title: t.text().notNull(), + meetingType: t.varchar({ length: 100 }), + startsAt: t.timestamp({ mode: "date", withTimezone: true }).notNull(), + timezone: t.varchar({ length: 64 }).notNull(), + location: t.text(), + status: t.varchar({ length: 100 }).notNull(), + isCancelled: t.boolean().notNull().default(false), + isAmended: t.boolean().notNull().default(false), + sourceUrl: t.text().notNull(), + videoUrl: t.text(), + documents: t + .jsonb() + .$type() + .notNull() + .default([]), + sourceVersion: t.varchar({ length: 100 }).notNull(), + contentHash: t.varchar({ length: 64 }).notNull(), + sourceUpdatedAt: t.timestamp({ mode: "date", withTimezone: true }).notNull(), + fetchedAt: t.timestamp({ mode: "date", withTimezone: true }) + .defaultNow() + .notNull(), + createdAt: t.timestamp().defaultNow().notNull(), + updatedAt: t + .timestamp({ mode: "date", withTimezone: true }) + .$onUpdateFn(() => sql`now()`), + }), + (table) => ({ + uniqueSourceMeeting: unique().on( + table.provider, + table.jurisdiction, + table.sourceId, + ), + startsAtIdx: index("local_government_meeting_starts_at_idx").on( + table.startsAt, + ), + }), +); + +export const LocalGovernmentMeetingItem = pgTable( + "local_government_meeting_item", + (t) => ({ + id: t.uuid().notNull().primaryKey().defaultRandom(), + provider: t.varchar({ length: 50 }).notNull(), + jurisdiction: t.varchar({ length: 100 }).notNull(), + sourceId: t.varchar({ length: 100 }).notNull(), + meetingSourceId: t.varchar({ length: 100 }).notNull(), + sequence: t.integer().notNull(), + agendaNumber: t.varchar({ length: 50 }), + title: t.text().notNull(), + agendaNote: t.text(), + minutesNote: t.text(), + isConsent: t.boolean().notNull().default(false), + action: t.text(), + actionText: t.text(), + outcome: t.varchar({ length: 100 }), + tally: t.varchar({ length: 100 }), + mover: t.varchar({ length: 256 }), + seconder: t.varchar({ length: 256 }), + documents: t + .jsonb() + .$type() + .notNull() + .default([]), + sourceVersion: t.varchar({ length: 100 }).notNull(), + contentHash: t.varchar({ length: 64 }).notNull(), + sourceUpdatedAt: t.timestamp({ mode: "date", withTimezone: true }).notNull(), + fetchedAt: t.timestamp({ mode: "date", withTimezone: true }) + .defaultNow() + .notNull(), + createdAt: t.timestamp().defaultNow().notNull(), + updatedAt: t + .timestamp({ mode: "date", withTimezone: true }) + .$onUpdateFn(() => sql`now()`), + }), + (table) => ({ + uniqueSourceItem: unique().on( + table.provider, + table.jurisdiction, + table.sourceId, + ), + meetingSourceIdx: index("local_government_item_meeting_source_idx").on( + table.provider, + table.jurisdiction, + table.meetingSourceId, + ), + }), +); + +export const LocalGovernmentVote = pgTable( + "local_government_vote", + (t) => ({ + id: t.uuid().notNull().primaryKey().defaultRandom(), + provider: t.varchar({ length: 50 }).notNull(), + jurisdiction: t.varchar({ length: 100 }).notNull(), + sourceId: t.varchar({ length: 100 }).notNull(), + itemSourceId: t.varchar({ length: 100 }).notNull(), + personSourceId: t.varchar({ length: 100 }), + personName: t.varchar({ length: 256 }).notNull(), + value: t.varchar({ length: 100 }).notNull(), + sort: t.integer().notNull().default(0), + sourceUpdatedAt: t.timestamp({ mode: "date", withTimezone: true }).notNull(), + fetchedAt: t.timestamp({ mode: "date", withTimezone: true }) + .defaultNow() + .notNull(), + createdAt: t.timestamp().defaultNow().notNull(), + updatedAt: t + .timestamp({ mode: "date", withTimezone: true }) + .$onUpdateFn(() => sql`now()`), + }), + (table) => ({ + uniqueSourceVote: unique().on( + table.provider, + table.jurisdiction, + table.sourceId, + ), + itemSourceIdx: index("local_government_vote_item_source_idx").on( + table.provider, + table.jurisdiction, + table.itemSourceId, + ), + }), +); + // Google Civic API response cache export const CivicApiCache = pgTable( "civic_api_cache", diff --git a/packages/env/src/registry.ts b/packages/env/src/registry.ts index 6686f519..63039e33 100644 --- a/packages/env/src/registry.ts +++ b/packages/env/src/registry.ts @@ -85,6 +85,7 @@ const scraperSourceLimitDefinitions = [ ["SCOTUS_MAX_ITEMS", "CourtListener opinion clusters per run.", "50"], ["SCC_CVIG_MAX_ITEMS", "Santa Clara voter-guide PDFs per run.", "10"], ["CA_SOS_MAX_ITEMS", "California SOS office pages per run.", "9"], + ["DURHAM_BOCC_MAX_ITEMS", "Durham County BOCC meetings per run.", "100"], ] as const; export const envRegistry = [