diff --git a/apps/scraper/README.md b/apps/scraper/README.md index 2e614c7..14dc63c 100644 --- a/apps/scraper/README.md +++ b/apps/scraper/README.md @@ -4,15 +4,16 @@ Pulls in government content like bills, court cases, and White House content and ## Active data sources -Only these five are registered and run by `all`: +Only these six are registered and run by `all`: -| CLI name | Source and data fetched | Stored/used as | -| ------------------- | ------------------------------------------------------------------------------------------ | ----------------------------------------------------------------------------------------------------- | -| `federalregister` | Federal Register API presidential documents, then each document's body HTML | `government_content`; AI article/summary and feed-image enrichment | -| `congress` | Congress.gov API bill list, detail, CRS summaries, formatted text, and legislative actions | `bill`; powers federal bill content and AI/feed enrichment | -| `scotus` | CourtListener opinion clusters, dockets, and sub-opinion text for the Supreme Court | `court_case`; powers court content and AI/feed enrichment | -| `scc-cvig` | Hand-configured Santa Clara County voter-guide PDFs | Candidate statements in `CivicApiCache`; the API matches statements to candidates | -| `ca-sos-statements` | California SOS statewide-office candidate-statement pages | Candidate statements in `CivicApiCache`; the API reads the cache and can fall back to the live source | +| CLI name | Source and data fetched | Stored/used as | +| -------------------- | ------------------------------------------------------------------------------------------ | ----------------------------------------------------------------------------------------------------- | +| `federalregister` | Federal Register API presidential documents, then each document's body HTML | `government_content`; AI article/summary and feed-image enrichment | +| `congress` | Congress.gov API bill list, detail, CRS summaries, formatted text, and legislative actions | `bill`; powers federal bill content and AI/feed enrichment | +| `scotus` | CourtListener opinion clusters, dockets, and sub-opinion text for the Supreme Court | `court_case`; powers court content and AI/feed enrichment | +| `scc-cvig` | Hand-configured Santa Clara County voter-guide PDFs | Candidate statements in `CivicApiCache`; the API matches statements to candidates | +| `ca-sos-statements` | California SOS statewide-office candidate-statement pages | Candidate statements in `CivicApiCache`; the API reads the cache and can fall back to the live source | +| `cedar-park-council` | Cedar Park's CivicEngage City Council page and its official Municode Meetings embed | Provider-neutral local meetings, documents, agenda items, motions, outcomes, and votes | `vote411`, `ca-lao-fiscal`, and `ca-vig-archive` remain under `src/scrapers/disabled/` and do not run. Their caches had no application @@ -124,6 +125,7 @@ CONGRESS_MAX_ITEMS=10 pnpm --filter @acme/scraper run start congress | `SCOTUS_MAX_ITEMS` | 50 | CourtListener opinion clusters | | `SCC_CVIG_MAX_ITEMS` | 10 | Voter-guide PDF documents | | `CA_SOS_MAX_ITEMS` | 9 | Statewide-office candidate-statement pages | +| `CEDAR_PARK_COUNCIL_MAX_ITEMS` | 100 | Council meetings (after the 12-month cutoff) | | `SCRAPER_MAX_NEW_ITEMS_PER_RUN` | 10 | New records receiving expensive AI/image enrichment | These are per-run limits, not durable calendar-day quotas. Schedule one run per @@ -132,6 +134,28 @@ invocation gets a fresh allowance. Source limits cap API/page work; `SCRAPER_MAX_NEW_ITEMS_PER_RUN` separately caps expensive enrichment while still storing additional raw records for later backfill. +## Cedar Park City Council (`civicengage.ts`) + +Cedar Park's public site is CivicEngage, but the City Council records page now +embeds a keyless Municode Meetings publish page. The adapter follows that +official embed, keeps a 12-month Council-only window, downloads at most two +documents concurrently, and deterministically parses PDF text. AI/vision is not +used. Agendas, packets, minutes, and later document URLs are versioned beneath +one provider-neutral meeting record; unchanged reruns produce the same natural +keys and checksums. + +```bash +pnpm --filter @acme/scraper run start cedar-park-council --max-items 1 +``` + +To add a second CivicEngage jurisdiction using the same kind of official +Municode embed, add a `CivicEngageJurisdictionConfig` beside +`cedarParkCouncilSource` with its CivicEngage host/path, IANA timezone, +Municode `cid`/`ppid`, and governing-body matcher, then instantiate the same +discovery/parser pipeline. If its records page uses Agenda Center, Legistar, or +another provider, implement that provider behind the same local-government +persistence contract instead. + --- ## Congress bills (`congress.ts`) diff --git a/apps/scraper/src/scraper-contracts.ts b/apps/scraper/src/scraper-contracts.ts index bfcfd04..f590547 100644 --- a/apps/scraper/src/scraper-contracts.ts +++ b/apps/scraper/src/scraper-contracts.ts @@ -1,6 +1,7 @@ import type { ScraperEnvContract } from "@acme/env"; import { caSosStatementsConfig } from "./scrapers/ca-sos-statements.config.js"; +import { cedarParkCouncilConfig } from "./scrapers/civicengage.config.js"; import { congressConfig } from "./scrapers/congress.config.js"; import { federalregisterConfig } from "./scrapers/federalregister.config.js"; import { sccCvigConfig } from "./scrapers/scc-cvig.config.js"; @@ -12,4 +13,5 @@ export const scraperContracts: readonly ScraperEnvContract[] = [ scotusConfig, sccCvigConfig, caSosStatementsConfig, + cedarParkCouncilConfig, ]; diff --git a/apps/scraper/src/scrapers.ts b/apps/scraper/src/scrapers.ts index d4ab8d4..9bbcfcc 100644 --- a/apps/scraper/src/scrapers.ts +++ b/apps/scraper/src/scrapers.ts @@ -1,5 +1,6 @@ import type { Scraper } from "./utils/types.js"; import { caSosStatements } from "./scrapers/ca-sos-statements.js"; +import { cedarParkCouncil } from "./scrapers/civicengage.js"; import { congress } from "./scrapers/congress.js"; import { federalregister } from "./scrapers/federalregister.js"; import { sccCvig } from "./scrapers/scc-cvig.js"; @@ -11,4 +12,5 @@ export const scrapers: readonly Scraper[] = [ scotus, sccCvig, caSosStatements, + cedarParkCouncil, ]; diff --git a/apps/scraper/src/scrapers/civicengage-parser.test.ts b/apps/scraper/src/scrapers/civicengage-parser.test.ts new file mode 100644 index 0000000..3b87f7a --- /dev/null +++ b/apps/scraper/src/scrapers/civicengage-parser.test.ts @@ -0,0 +1,115 @@ +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import test from "node:test"; + +import { + parseAgendaItems, + parseCentralMeetingDate, + parseMunicodePublishPage, +} from "./civicengage-parser.js"; +import { cedarParkCouncilSource } from "./civicengage.config.js"; + +const fixture = (name: string) => + readFile(new URL(`./fixtures/civicengage/${name}`, import.meta.url), "utf8"); + +test("parses regular, special, cancelled, and amended meeting records", async () => { + const meetings = parseMunicodePublishPage( + await fixture("meetings.html"), + cedarParkCouncilSource, + { now: new Date("2026-03-01T00:00:00Z") }, + ); + assert.equal(meetings.length, 4); + assert.deepEqual( + meetings.map(({ title, meetingType, status }) => ({ + title, + meetingType, + status, + })), + [ + { + title: "City Council Mtg. - CANCELLED", + meetingType: "regular", + status: "cancelled", + }, + { + title: "City Council - Special Called", + meetingType: "special", + status: "held", + }, + { + title: "City Council Mtg.", + meetingType: "regular", + status: "completed", + }, + { + title: "City Council Mtg. - Amended", + meetingType: "regular", + status: "amended", + }, + ], + ); + assert.match(meetings[2]?.externalId ?? "", /^[a-f0-9]{32}$/); + assert.equal( + meetings[2]?.documents[0]?.url, + "https://mccmeetings.blob.core.usgovcloudapi.net/cptx-pubu/MEET-Agenda-2c1128fc619f48d6bf29cb94897a2d7f.pdf", + ); + assert.equal( + meetings[2]?.canonicalUrl, + "https://www.cedarparktexas.gov/596/City-Council-Agendas", + ); + assert.deepEqual( + meetings[3]?.documents.map(({ isCurrent }) => isCurrent), + [true, false], + ); +}); + +test("applies a twelve-month cutoff and parses Central daylight time", async () => { + const meetings = parseMunicodePublishPage( + await fixture("meetings.html"), + cedarParkCouncilSource, + { + now: new Date("2026-07-21T12:00:00Z"), + cutoff: new Date("2025-07-21T12:00:00Z"), + }, + ); + assert.equal(meetings.length, 3); + assert.equal( + parseCentralMeetingDate("2/12/2026", "6:00 PM").toISOString(), + "2026-02-13T00:00:00.000Z", + ); + assert.equal( + parseCentralMeetingDate("7/9/2026", "7:00 PM").toISOString(), + "2026-07-10T00:00:00.000Z", + ); +}); + +test("deterministically parses items, motions, outcomes, tallies, and roll calls", async () => { + const agenda = await fixture("agenda.txt"); + const minutes = await fixture("minutes.txt"); + const sourceUrl = "https://official.example/agenda.pdf"; + const first = parseAgendaItems(agenda, minutes, sourceUrl); + const second = parseAgendaItems(agenda, minutes, sourceUrl); + assert.deepEqual(first, second); + assert.equal(first.length, 6); + + const ordinance = first.find((item) => item.itemNumber === "E.1"); + assert.equal(ordinance?.itemType, "ordinance"); + assert.equal(ordinance?.consent, true); + assert.equal(ordinance?.outcome, "approved"); + + const resolution = first.find((item) => item.itemNumber === "F.1"); + assert.equal( + resolution?.motion, + "Motion to approve Agenda Item F.1 as presented.", + ); + assert.match(resolution?.voteSummary ?? "", /^6-0/); + assert.equal(resolution?.outcome, "approved"); + + const rollCall = first.find((item) => item.itemNumber === "H.1"); + assert.equal(rollCall?.votes.length, 6); + assert.deepEqual(rollCall?.votes.at(-1), { voterName: "Darby", value: "no" }); + assert.equal(rollCall?.sourceUrl, sourceUrl); + + const noAction = first.find((item) => item.itemNumber === "H.2"); + assert.equal(noAction?.outcome, "no_action"); +}); diff --git a/apps/scraper/src/scrapers/civicengage-parser.ts b/apps/scraper/src/scrapers/civicengage-parser.ts new file mode 100644 index 0000000..4f5760e --- /dev/null +++ b/apps/scraper/src/scrapers/civicengage-parser.ts @@ -0,0 +1,414 @@ +import { createHash } from "node:crypto"; +import * as cheerio from "cheerio"; + +import type { CivicEngageJurisdictionConfig } from "./civicengage.config.js"; + +export type MeetingType = + | "regular" + | "special" + | "work_session" + | "retreat" + | "notice" + | "other"; +export type MeetingStatus = + | "scheduled" + | "held" + | "completed" + | "cancelled" + | "amended"; +export type DocumentType = "agenda" | "packet" | "minutes" | "html"; + +export interface DiscoveredDocument { + type: DocumentType; + title: string; + url: string; + mediaType: string; + isCurrent: boolean; +} + +export interface DiscoveredMeeting { + externalId: string; + title: string; + meetingType: MeetingType; + status: MeetingStatus; + startsAt: Date; + location?: string; + canonicalUrl: string; + documents: DiscoveredDocument[]; +} + +export interface ParsedVote { + voterName: string; + value: string; +} + +export interface ParsedAgendaItem { + externalId: string; + sequence: number; + itemNumber: string; + section?: string; + itemType: string; + title: string; + description?: string; + consent: boolean; + motion?: string; + outcome?: string; + voteSummary?: string; + sourceUrl: string; + votes: ParsedVote[]; +} + +const SOURCE_VERSION = "civicengage-municode-v1"; +export { SOURCE_VERSION }; + +function clean(value: string): string { + return value.replace(/\s+/g, " ").trim(); +} + +function directDocumentUrl(href: string, baseUrl: string): string { + const url = new URL(href, baseUrl); + if (url.hostname === "meetings.municode.com" && url.pathname === "/d/f") { + const target = url.searchParams.get("u"); + if (target) return new URL(target).toString(); + } + return url.toString(); +} + +function meetingType(title: string): MeetingType { + if (/notice\s+of\s+(?:a\s+)?possible\s+quorum/i.test(title)) return "notice"; + if (/retreat/i.test(title)) return "retreat"; + if (/work\s*shop|work\s+session/i.test(title)) return "work_session"; + if (/special\s+(?:called\s+)?(?:meeting|mtg)|special\s+called/i.test(title)) { + return "special"; + } + if (/council\s+(?:meeting|mtg)|regular/i.test(title)) return "regular"; + return "other"; +} + +function meetingStatus( + title: string, + startsAt: Date, + hasMinutes: boolean, + now: Date, +): MeetingStatus { + if (/cancel(?:led|ed|lation)/i.test(title)) return "cancelled"; + if (/amend(?:ed|ment)/i.test(title)) return "amended"; + if (hasMinutes) return "completed"; + return startsAt.getTime() > now.getTime() ? "scheduled" : "held"; +} + +export function parseMeetingDate( + dateText: string, + timeText: string, + timezone: string, +): Date { + const match = dateText.trim().match(/^(\d{1,2})\/(\d{1,2})\/(\d{4})$/); + const time = timeText.trim().match(/^(\d{1,2}):(\d{2})\s*([AP]M)$/i); + if (!match || !time) + throw new Error(`Unrecognized meeting date: ${dateText} ${timeText}`); + const month = Number(match[1]); + const day = Number(match[2]); + const year = Number(match[3]); + let hour = Number(time[1]) % 12; + if (time[3]?.toUpperCase() === "PM") hour += 12; + const minute = Number(time[2]); + const desiredUtc = Date.UTC(year, month - 1, day, hour, minute); + // Intl gives the local wall-clock parts for a UTC guess. The delta converts + // the desired local clock time to its real instant, including DST. + let instant = desiredUtc; + for (let pass = 0; pass < 2; pass++) { + const parts = new Intl.DateTimeFormat("en-US", { + timeZone: timezone, + year: "numeric", + month: "2-digit", + day: "2-digit", + hour: "2-digit", + minute: "2-digit", + hourCycle: "h23", + }).formatToParts(new Date(instant)); + const part = (type: Intl.DateTimeFormatPartTypes) => + Number(parts.find((value) => value.type === type)?.value); + const representedUtc = Date.UTC( + part("year"), + part("month") - 1, + part("day"), + part("hour"), + part("minute"), + ); + instant += desiredUtc - representedUtc; + } + return new Date(instant); +} + +export function parseCentralMeetingDate( + dateText: string, + timeText: string, +): Date { + return parseMeetingDate(dateText, timeText, "America/Chicago"); +} + +function stableMeetingId(title: string, startsAt: Date): string { + const normalizedTitle = title + .replace(/\b(?:amended|cancelled|canceled)\b/gi, "") + .replace(/[^a-z0-9]+/gi, " ") + .trim() + .toLowerCase(); + return createHash("sha256") + .update(`${startsAt.toISOString()}|${normalizedTitle}`) + .digest("hex") + .slice(0, 32); +} + +export function parseMunicodePublishPage( + html: string, + config: CivicEngageJurisdictionConfig, + options: { now?: Date; cutoff?: Date } = {}, +): DiscoveredMeeting[] { + const $ = cheerio.load(html); + const now = options.now ?? new Date(); + const canonicalUrl = new URL( + config.recordsPagePath, + config.civicEngageBaseUrl, + ).toString(); + const meetings: DiscoveredMeeting[] = []; + + $("table tr").each((_index, row) => { + const $row = $(row); + const title = clean($row.find("td.meeting").text()); + const dateText = clean($row.find("td.date").text()); + const timeText = clean($row.find("td.time").text()); + if ( + !title || + !dateText || + !timeText || + !config.provider.meetingNamePattern.test(title) + ) + return; + + let startsAt: Date; + try { + startsAt = parseMeetingDate(dateText, timeText, config.timezone); + } catch { + return; + } + if (options.cutoff && startsAt < options.cutoff) return; + + const documents: DiscoveredDocument[] = []; + for (const type of ["agenda", "packet", "minutes"] as const) { + const link = $row.find(`td.${type} a`).first(); + const href = link.attr("href"); + if (!href) continue; + documents.push({ + type, + title: clean(link.find("img").attr("alt") ?? `${type} for ${title}`), + url: directDocumentUrl(href, "https://meetings.municode.com"), + mediaType: "application/pdf", + isCurrent: true, + }); + } + + meetings.push({ + externalId: stableMeetingId(title, startsAt), + title, + meetingType: meetingType(title), + status: meetingStatus( + title, + startsAt, + documents.some((document) => document.type === "minutes"), + now, + ), + startsAt, + location: clean($row.find("td.venue").text()) || undefined, + canonicalUrl, + documents, + }); + }); + + const grouped = new Map(); + for (const candidate of meetings) { + const existing = grouped.get(candidate.externalId); + if (!existing) { + grouped.set(candidate.externalId, candidate); + continue; + } + const candidatePreferred = + ["amended", "cancelled"].includes(candidate.status) && + !["amended", "cancelled"].includes(existing.status); + const preferred = candidatePreferred ? candidate : existing; + const secondary = candidatePreferred ? existing : candidate; + const preferredTypes = new Set( + preferred.documents.map((document) => document.type), + ); + const documents = [ + ...preferred.documents.map((document) => ({ + ...document, + isCurrent: true, + })), + ...secondary.documents.map((document) => ({ + ...document, + isCurrent: !preferredTypes.has(document.type), + })), + ].filter( + (document, index, all) => + all.findIndex( + (candidateDocument) => candidateDocument.url === document.url, + ) === index, + ); + grouped.set(candidate.externalId, { ...preferred, documents }); + } + + return [...grouped.values()].sort( + (a, b) => b.startsAt.getTime() - a.startsAt.getTime(), + ); +} + +const SECTION_NAMES = [ + "Consent Agenda", + "Public Hearings", + "Regular Agenda (Non-Consent)", + "Regular Agenda", + "Executive Session", + "Open Meeting", +]; + +function normalizedLines(text: string): string[] { + return text + .replace(/\r/g, "") + .split("\n") + .map((line) => clean(line)) + .filter(Boolean) + .filter((line) => !/^City Council (?:Agenda|Minutes)$/i.test(line)) + .filter( + (line) => + !/^(?:January|February|March|April|May|June|July|August|September|October|November|December) \d{1,2}, \d{4}$/i.test( + line, + ), + ); +} + +interface TextItemBlock { + number: string; + text: string; + section?: string; +} + +function itemBlocks(text: string): TextItemBlock[] { + const lines = normalizedLines(text); + const blocks: TextItemBlock[] = []; + let section: string | undefined; + let current: TextItemBlock | undefined; + + for (const line of lines) { + const sectionName = SECTION_NAMES.find( + (candidate) => candidate.toLowerCase() === line.toLowerCase(), + ); + if (sectionName) { + section = sectionName; + continue; + } + const start = line.match(/^([A-Z]{1,2}\.\d+)\s+(.+)$/); + if (start) { + if (current) blocks.push(current); + current = { number: start[1]!, text: start[2]!, section }; + } else if (current && !/^(?:Page \d+|-{3,})$/i.test(line)) { + current.text += `\n${line}`; + } + } + if (current) blocks.push(current); + return blocks; +} + +function itemType(text: string, section?: string): string { + if (/public hearing/i.test(text) || /public hearings?/i.test(section ?? "")) + return "public_hearing"; + if (/\bordinance\b/i.test(text)) return "ordinance"; + if (/\bresolution\b/i.test(text)) return "resolution"; + if (/\bminutes\b/i.test(text)) return "minutes"; + if (/presentation|proclamation/i.test(text)) return "presentation"; + if (/executive session/i.test(section ?? "")) return "executive_session"; + return "agenda_item"; +} + +function firstMatch(text: string, pattern: RegExp): string | undefined { + return clean(text.match(pattern)?.[1] ?? "") || undefined; +} + +function parseRollCall(text: string): ParsedVote[] { + const votes: ParsedVote[] = []; + const patterns: [RegExp, string][] = [ + [/(?:Voting|Voted)\s+(?:Aye|Yes):\s*([^\n]+)/gi, "yes"], + [/(?:Voting|Voted)\s+(?:Nay|No):\s*([^\n]+)/gi, "no"], + [/Abstain(?:ing)?:\s*([^\n]+)/gi, "abstain"], + [/Absent:\s*([^\n]+)/gi, "absent"], + ]; + for (const [pattern, value] of patterns) { + let match: RegExpExecArray | null; + while ((match = pattern.exec(text)) !== null) { + for (const rawName of (match[1] ?? "").split(/,|\band\b/i)) { + const voterName = clean(rawName.replace(/Council(?:member)?/gi, "")); + if (voterName) votes.push({ voterName, value }); + } + } + } + return votes; +} + +function outcome(text: string, voteSummary?: string): string | undefined { + if (/no action taken/i.test(text)) return "no_action"; + if (/approved under the consent agenda|motion (?:carried|passed)/i.test(text)) + return "approved"; + const tally = voteSummary?.match(/(\d+)\s*[-–]\s*(\d+)/); + if (tally) return Number(tally[1]) > Number(tally[2]) ? "approved" : "failed"; + return undefined; +} + +export function parseAgendaItems( + agendaText: string, + minutesText: string | undefined, + sourceUrl: string, +): ParsedAgendaItem[] { + const agenda = itemBlocks(agendaText); + const minutes = new Map( + itemBlocks(minutesText ?? "").map((block) => [block.number, block]), + ); + + return agenda.map((block, index) => { + const minutesBlock = minutes.get(block.number); + const combined = minutesBlock?.text ?? block.text; + const parsedMotion = firstMatch( + combined, + /(Motion to[\s\S]*?)(?=\n(?:Movant|Second|Vote):|$)/i, + ); + const motion = + parsedMotion && + /Agenda Items?/i.test(parsedMotion) && + !new RegExp(`\\b${block.number.replace(".", "\\.")}\\b`, "i").test( + parsedMotion, + ) && + !/consent agenda/i.test(block.section ?? "") + ? undefined + : parsedMotion; + const voteSummary = firstMatch(combined, /Vote:\s*([^\n]+)/i); + const title = clean(block.text.split("\n")[0] ?? block.text); + const description = clean( + block.text.replace(block.text.split("\n")[0] ?? "", ""), + ); + return { + externalId: block.number.toLowerCase(), + sequence: index + 1, + itemNumber: block.number, + section: block.section, + itemType: itemType(block.text, block.section), + title, + description: description || undefined, + consent: + /consent agenda/i.test(block.section ?? "") || + /approved under the consent agenda/i.test(combined), + motion, + outcome: outcome(combined, voteSummary), + voteSummary, + sourceUrl, + votes: parseRollCall(combined), + }; + }); +} diff --git a/apps/scraper/src/scrapers/civicengage.config.ts b/apps/scraper/src/scrapers/civicengage.config.ts new file mode 100644 index 0000000..0bb5bef --- /dev/null +++ b/apps/scraper/src/scrapers/civicengage.config.ts @@ -0,0 +1,57 @@ +import type { ScraperEnvContract } from "@acme/env"; + +export interface CivicEngageJurisdictionConfig { + id: string; + name: string; + governingBody: string; + civicEngageBaseUrl: string; + recordsPagePath: string; + timezone: string; + provider: { + kind: "municode-publish-page"; + clientId: string; + publishPageId: string; + meetingNamePattern: RegExp; + }; +} + +/** + * Cedar Park's public website is CivicEngage, but its City Council records page + * now embeds Municode Meetings. Keeping both halves in configuration makes the + * adapter reusable when another CivicEngage jurisdiction uses the same embed, + * without pretending all CivicEngage Agenda Centers share this provider. + */ +export const cedarParkCouncilSource: CivicEngageJurisdictionConfig = { + id: "cedar-park-tx", + name: "City of Cedar Park, Texas", + governingBody: "City Council", + civicEngageBaseUrl: "https://www.cedarparktexas.gov", + recordsPagePath: "/596/City-Council-Agendas", + timezone: "America/Chicago", + provider: { + kind: "municode-publish-page", + clientId: "CPTX", + publishPageId: "d5927f56-2e55-4a02-a095-4ed5e6109cfd", + meetingNamePattern: /\b(?:city\s+)?council\b/i, + }, +}; + +export function municodePublishPageUrl( + config: CivicEngageJurisdictionConfig, +): string { + const url = new URL("https://meetings.municode.com/PublishPage/index"); + url.searchParams.set("cid", config.provider.clientId); + url.searchParams.set("ppid", config.provider.publishPageId); + url.searchParams.set("p", "-1"); + return url.toString(); +} + +export const cedarParkCouncilConfig = { + id: "cedar-park-council", + name: "Cedar Park City Council", + source: "Cedar Park CivicEngage / official Municode Meetings embed", + environment: { + required: ["POSTGRES_URL"], + optional: ["CEDAR_PARK_COUNCIL_MAX_ITEMS"], + }, +} as const satisfies ScraperEnvContract; diff --git a/apps/scraper/src/scrapers/civicengage.ts b/apps/scraper/src/scrapers/civicengage.ts new file mode 100644 index 0000000..dec3f12 --- /dev/null +++ b/apps/scraper/src/scrapers/civicengage.ts @@ -0,0 +1,353 @@ +import { createHash } from "node:crypto"; +import pLimit from "p-limit"; +import { getDocumentProxy } from "unpdf"; + +import { and, eq, notInArray } from "@acme/db"; +import { db } from "@acme/db/client"; +import { + LocalGovernmentAgendaItem, + LocalGovernmentDocument, + LocalGovernmentMeeting, + LocalGovernmentVote, +} from "@acme/db/schema"; + +import type { Scraper } from "../utils/types.js"; +import type { + DiscoveredDocument, + DiscoveredMeeting, + ParsedAgendaItem, +} from "./civicengage-parser.js"; +import { setExpectedTotal } from "../utils/db/metrics.js"; +import { fetchWithRetry } from "../utils/fetch.js"; +import { createLogger } from "../utils/log.js"; +import { + parseAgendaItems, + parseMunicodePublishPage, + SOURCE_VERSION, +} from "./civicengage-parser.js"; +import { + cedarParkCouncilConfig, + cedarParkCouncilSource, + municodePublishPageUrl, +} from "./civicengage.config.js"; + +const logger = createLogger("cedar-park-council"); +const documentLimit = pLimit(2); +const USER_AGENT = + "Billion civic data scraper (+https://github.com/billion-app/billion)"; + +interface FetchedDocument extends DiscoveredDocument { + checksum?: string; + extractedText?: string; + fetchedAt?: Date; +} + +function checksum(bytes: Uint8Array): string { + return createHash("sha256").update(bytes).digest("hex"); +} + +export async function extractPdfText(bytes: Uint8Array): Promise { + const proxy = await getDocumentProxy(bytes); + const pages: string[] = []; + for (let pageNumber = 1; pageNumber <= proxy.numPages; pageNumber++) { + const page = await proxy.getPage(pageNumber); + const content = await page.getTextContent(); + const items = content.items + .map((raw) => { + const item = raw as { str?: string; transform?: number[] }; + return item.str && item.transform + ? { + text: item.str, + x: item.transform[4] ?? 0, + y: item.transform[5] ?? 0, + } + : null; + }) + .filter( + (item): item is { text: string; x: number; y: number } => item !== null, + ) + .sort((a, b) => (Math.abs(a.y - b.y) > 2 ? b.y - a.y : a.x - b.x)); + + const lines: { y: number; parts: string[] }[] = []; + for (const item of items) { + const line = lines.find( + (candidate) => Math.abs(candidate.y - item.y) <= 2, + ); + if (line) line.parts.push(item.text); + else lines.push({ y: item.y, parts: [item.text] }); + } + pages.push(lines.map((line) => line.parts.join(" ")).join("\n")); + } + return pages.join("\n\n"); +} + +async function fetchDocument( + document: DiscoveredDocument, +): Promise { + const [cached] = await db + .select({ + checksum: LocalGovernmentDocument.checksum, + extractedText: LocalGovernmentDocument.extractedText, + fetchedAt: LocalGovernmentDocument.fetchedAt, + }) + .from(LocalGovernmentDocument) + .where(eq(LocalGovernmentDocument.url, document.url)) + .limit(1); + if (cached?.checksum) { + return { + ...document, + checksum: cached.checksum, + extractedText: cached.extractedText ?? undefined, + fetchedAt: cached.fetchedAt ?? undefined, + }; + } + + try { + const response = await fetchWithRetry(document.url, { + headers: { Accept: document.mediaType, "User-Agent": USER_AGENT }, + timeoutMs: 60_000, + }); + const bytes = new Uint8Array(await response.arrayBuffer()); + const shouldExtract = + document.type === "agenda" || document.type === "minutes"; + return { + ...document, + checksum: checksum(bytes), + extractedText: shouldExtract ? await extractPdfText(bytes) : undefined, + fetchedAt: new Date(), + }; + } catch (error) { + logger.warn(`Could not fetch ${document.type} ${document.url}`, error); + return document; + } +} + +function meetingHash( + meeting: DiscoveredMeeting, + documents: FetchedDocument[], + items: ParsedAgendaItem[], +): string { + return createHash("sha256") + .update( + JSON.stringify({ + title: meeting.title, + meetingType: meeting.meetingType, + status: meeting.status, + startsAt: meeting.startsAt.toISOString(), + location: meeting.location, + documents: documents.map(({ type, url, checksum, isCurrent }) => ({ + type, + url, + checksum, + isCurrent, + })), + items, + }), + ) + .digest("hex"); +} + +async function persistMeeting( + meeting: DiscoveredMeeting, + documents: FetchedDocument[], + items: ParsedAgendaItem[], +): Promise { + const fetchedAt = new Date(); + const contentHash = meetingHash(meeting, documents, items); + + await db.transaction(async (tx) => { + const [row] = await tx + .insert(LocalGovernmentMeeting) + .values({ + source: "civicengage", + sourceVersion: SOURCE_VERSION, + jurisdiction: cedarParkCouncilSource.id, + governingBody: cedarParkCouncilSource.governingBody, + externalId: meeting.externalId, + title: meeting.title, + meetingType: meeting.meetingType, + status: meeting.status, + startsAt: meeting.startsAt, + location: meeting.location, + canonicalUrl: meeting.canonicalUrl, + contentHash, + fetchedAt, + }) + .onConflictDoUpdate({ + target: [ + LocalGovernmentMeeting.source, + LocalGovernmentMeeting.jurisdiction, + LocalGovernmentMeeting.externalId, + ], + set: { + sourceVersion: SOURCE_VERSION, + title: meeting.title, + meetingType: meeting.meetingType, + status: meeting.status, + startsAt: meeting.startsAt, + location: meeting.location, + canonicalUrl: meeting.canonicalUrl, + contentHash, + fetchedAt, + updatedAt: fetchedAt, + }, + }) + .returning({ id: LocalGovernmentMeeting.id }); + if (!row) + throw new Error(`Failed to persist meeting ${meeting.externalId}`); + + await tx + .update(LocalGovernmentDocument) + .set({ isCurrent: false, updatedAt: fetchedAt }) + .where(eq(LocalGovernmentDocument.meetingId, row.id)); + + for (const document of documents) { + await tx + .insert(LocalGovernmentDocument) + .values({ + meetingId: row.id, + type: document.type, + title: document.title, + url: document.url, + mediaType: document.mediaType, + checksum: document.checksum, + extractedText: document.extractedText, + isCurrent: document.isCurrent, + fetchedAt: document.fetchedAt, + }) + .onConflictDoUpdate({ + target: [ + LocalGovernmentDocument.meetingId, + LocalGovernmentDocument.type, + LocalGovernmentDocument.url, + ], + set: { + title: document.title, + mediaType: document.mediaType, + checksum: document.checksum, + extractedText: document.extractedText, + isCurrent: document.isCurrent, + fetchedAt: document.fetchedAt, + updatedAt: fetchedAt, + }, + }); + } + + for (const item of items) { + const [itemRow] = await tx + .insert(LocalGovernmentAgendaItem) + .values({ + meetingId: row.id, + externalId: item.externalId, + sequence: item.sequence, + itemNumber: item.itemNumber, + section: item.section, + itemType: item.itemType, + title: item.title, + description: item.description, + consent: item.consent, + motion: item.motion, + outcome: item.outcome, + voteSummary: item.voteSummary, + sourceUrl: item.sourceUrl, + }) + .onConflictDoUpdate({ + target: [ + LocalGovernmentAgendaItem.meetingId, + LocalGovernmentAgendaItem.externalId, + ], + set: { + sequence: item.sequence, + itemNumber: item.itemNumber, + section: item.section, + itemType: item.itemType, + title: item.title, + description: item.description, + consent: item.consent, + motion: item.motion, + outcome: item.outcome, + voteSummary: item.voteSummary, + sourceUrl: item.sourceUrl, + updatedAt: fetchedAt, + }, + }) + .returning({ id: LocalGovernmentAgendaItem.id }); + if (!itemRow) continue; + await tx + .delete(LocalGovernmentVote) + .where(eq(LocalGovernmentVote.agendaItemId, itemRow.id)); + if (item.votes.length === 0) continue; + await tx.insert(LocalGovernmentVote).values( + item.votes.map((vote) => ({ + agendaItemId: itemRow.id, + voterName: vote.voterName, + value: vote.value, + })), + ); + } + const staleItemFilter = + items.length === 0 + ? eq(LocalGovernmentAgendaItem.meetingId, row.id) + : and( + eq(LocalGovernmentAgendaItem.meetingId, row.id), + notInArray( + LocalGovernmentAgendaItem.externalId, + items.map((item) => item.externalId), + ), + ); + await tx.delete(LocalGovernmentAgendaItem).where(staleItemFilter); + }); +} + +async function processMeeting(meeting: DiscoveredMeeting): Promise { + const documents = await Promise.all( + meeting.documents.map((document) => + documentLimit(() => fetchDocument(document)), + ), + ); + const agenda = documents.find((document) => document.type === "agenda"); + const minutes = documents.find((document) => document.type === "minutes"); + const items = agenda?.extractedText + ? parseAgendaItems(agenda.extractedText, minutes?.extractedText, agenda.url) + : []; + await persistMeeting(meeting, documents, items); + logger.success( + `${meeting.startsAt.toISOString().slice(0, 10)} ${meeting.title}: ${items.length} items`, + ); +} + +async function scrape(maxItems = 100): Promise { + const now = new Date(); + const cutoff = new Date(now); + cutoff.setUTCFullYear(cutoff.getUTCFullYear() - 1); + const listingUrl = municodePublishPageUrl(cedarParkCouncilSource); + logger.info( + `Discovering Cedar Park Council meetings since ${cutoff.toISOString()}`, + ); + const response = await fetchWithRetry(listingUrl, { + headers: { Accept: "text/html", "User-Agent": USER_AGENT }, + }); + const meetings = parseMunicodePublishPage( + await response.text(), + cedarParkCouncilSource, + { + now, + cutoff, + }, + ).slice(0, maxItems); + setExpectedTotal(meetings.length); + + // Meetings are intentionally sequential; only document requests within one + // meeting run concurrently, capped at two against the public records host. + for (const meeting of meetings) await processMeeting(meeting); + logger.success(`Completed ${meetings.length} Cedar Park Council meetings`); +} + +export const cedarParkCouncil: Scraper = { + ...cedarParkCouncilConfig, + scrape: (options) => + scrape( + (options?.maxItems ?? Number(process.env.CEDAR_PARK_COUNCIL_MAX_ITEMS)) || + 100, + ), +}; diff --git a/apps/scraper/src/scrapers/fixtures/civicengage/agenda.txt b/apps/scraper/src/scrapers/fixtures/civicengage/agenda.txt new file mode 100644 index 0000000..b83450c --- /dev/null +++ b/apps/scraper/src/scrapers/fixtures/civicengage/agenda.txt @@ -0,0 +1,10 @@ +CITY COUNCIL AGENDA +Consent Agenda +D.1 Approval Of Minutes From The Regular Scheduled City Council Meeting On December 18, 2025. +E.1 Second Reading And Approval Of An Ordinance Calling And Ordering A Special Called Election To Be Held On May 2, 2026. +F.1 A Resolution To Adopt The Cedar Park Transportation Criteria Manual (TCM). +Public Hearings +G.1 First Reading And Public Hearing Of An Ordinance For A Future Land Use Plan Amendment. +Regular Agenda (Non-Consent) +H.1 Consideration Of A Resolution Authorizing A Grant Agreement With The Lower Colorado River Authority In The Amount Of $100,000. +H.2 Consider Action, If Any, On Items Discussed In Executive Session. diff --git a/apps/scraper/src/scrapers/fixtures/civicengage/meetings.html b/apps/scraper/src/scrapers/fixtures/civicengage/meetings.html new file mode 100644 index 0000000..c907d64 --- /dev/null +++ b/apps/scraper/src/scrapers/fixtures/civicengage/meetings.html @@ -0,0 +1,44 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
City Council Mtg.2/12/20266:00 PMCEDAR PARK CITY HALL COUNCIL CHAMBERSPDF Agenda for February 12, 2026 City Council Mtg.PDF Packet for February 12, 2026 City Council Mtg.PDF Minutes for February 12, 2026 City Council Mtg.
City Council - Special Called2/25/20265:00 PMCITY HALLAgenda
City Council Mtg. - CANCELLED3/12/20266:00 PMCITY HALLAgenda
City Council Mtg. - Amended3/13/20257:00 PMCITY HALLAgenda
City Council Mtg.3/13/20257:00 PMCITY HALLPrevious Agenda
Planning and Zoning Commission3/1/20266:00 PMCITY HALL
diff --git a/apps/scraper/src/scrapers/fixtures/civicengage/minutes.txt b/apps/scraper/src/scrapers/fixtures/civicengage/minutes.txt new file mode 100644 index 0000000..32f53f9 --- /dev/null +++ b/apps/scraper/src/scrapers/fixtures/civicengage/minutes.txt @@ -0,0 +1,24 @@ +CITY COUNCIL MINUTES +Consent Agenda +D.1 Approval Of Minutes From The Regular Scheduled City Council Meeting On December 18, 2025. +Approved under the Consent Agenda. +E.1 Second Reading And Approval Of An Ordinance Calling And Ordering A Special Called Election To Be Held On May 2, 2026. +Approved under the Consent Agenda. +F.1 A Resolution To Adopt The Cedar Park Transportation Criteria Manual (TCM). +Motion to approve Agenda Item F.1 as presented. +Movant: Councilmember Kirkland +Second: Councilmember Frezza +Vote: 6-0 with Mayor Pro Tem Boyce absent from meeting +Public Hearings +G.1 First Reading And Public Hearing Of An Ordinance For A Future Land Use Plan Amendment. +Mayor opened the Public Hearing. No Public Comment. Mayor closed the Public Hearing. +Regular Agenda (Non-Consent) +H.1 Consideration Of A Resolution Authorizing A Grant Agreement With The Lower Colorado River Authority In The Amount Of $100,000. +Motion to approve Agenda Item H.1 as presented. +Movant: Councilmember Kirkland +Second: Councilmember Duffy +Vote: 5-1. +Voting Aye: Kirkland, Duffy, Harris, Frezza, and Penniman-Morin +Voting Nay: Darby +H.2 Consider Action, If Any, On Items Discussed In Executive Session. +No action taken. diff --git a/docs/civic-data-sources.md b/docs/civic-data-sources.md index ba12620..82c9400 100644 --- a/docs/civic-data-sources.md +++ b/docs/civic-data-sources.md @@ -9,6 +9,7 @@ How to obtain keys/access for every civic integration. For local dev, copy `.env | Google Places (address autocomplete) | Yes | Pay-as-you-go | `GOOGLE_PLACES_API_KEY` (→ `GOOGLE_API_KEY` → `GOOGLE_CIVIC_API_KEY`) | | Vote Smart | Yes | Free (org tier) | `VOTE_SMART_API_KEY` | | Legistar (local councils) | No | Free | — | +| Cedar Park council records | No | Free | — | | VOTE411 / LWV (scraper) | No | Free | — | | CA SOS Voter Guide (scraper) | No | Free | — | | Santa Clara measure pipeline (scraper) | No | Free | — | @@ -27,6 +28,15 @@ How to obtain keys/access for every civic integration. For local dev, copy `.env **Legistar Web API** — local council meetings, legislation, votes, agendas; no key, unlimited public data. Supported jurisdictions (client id): San Jose (`sanjose`), Santa Clara County (`santaclara`), Sunnyvale (`sunnyvale`). Add a city by extracting its `*.legistar.com` subdomain into the `JURISDICTIONS` constant in `packages/api/src/integrations/legistar.ts`. Base: (OData-compatible). +**Cedar Park City Council** — the official CivicEngage City Council page embeds +a Municode Meetings publish page. The `cedar-park-council` scraper follows that +embed for the latest 12 months, uses deterministic HTML/PDF parsing, and writes +the provider-neutral local-government tables read by the `localGovernment` +tRPC router. No AI key is needed. Configuration lives in +`apps/scraper/src/scrapers/civicengage.config.ts`; a second jurisdiction using +the same embed supplies a new CivicEngage host/path, timezone, Municode +`cid`/`ppid`, and body matcher. + **VOTE411 / League of Women Voters** — nonpartisan voter guides, candidate questionnaires, measure explanations; no key (scraper, rate-limited + cached). ⚠️ ToS bars commercial use without **written consent** and prohibits automated queries — a negotiated partnership is the only compliant path (tracked in the Outreach Tracker). Also `cavotes.org/easy-voter-guide/` for CA. **CA SOS Official Voter Guide (scraper)** — official statewide proposition summaries (AG), pro/con arguments, full-text links; no key. Source `voterguide.sos.ca.gov`. Distinct from the (dead) results API — this carries the human-written official measure content and feeds the cross-validation engine (`measure-sources/ca-sos-voterguide.ts`). CA statewide only; local measures use the county pipelines. diff --git a/docs/data-sources-api.md b/docs/data-sources-api.md index fd61ad4..265ab10 100644 --- a/docs/data-sources-api.md +++ b/docs/data-sources-api.md @@ -8,16 +8,16 @@ All live-API functions come from `@acme/api`. Scrapers run as CLI jobs in `apps/ ## Quick index -| Category | What you get | Import | -|---|---|---| -| [Ballot & elections](#ballot--elections) | Contests, candidates, polling places, ballot measures | `@acme/api` | -| [Ballot-measure enrichment](#ballot-measure-enrichment) | Summaries, fiscal impact, pro/con arguments, citations | `@acme/api` | -| [State legislation](#state-legislation) | CA bills, legislators, votes | `@acme/api` | -| [Local government](#local-government) | City/county meetings, ordinances, votes | `@acme/api` | -| [CA election results](#ca-election-results) | Statewide vote counts (live JSON feed) | `@acme/api` | -| [Address autocomplete](#address-autocomplete) | Street-address suggestions + place details | `@acme/api` | -| [Corpus content (DB)](#corpus-content-db) | Bills, court cases, government content articles | tRPC `content` router | -| [Scrapers (background jobs)](#scrapers-background-jobs) | Congress bills, SCOTUS, Federal Register, LAO, CA SOS | `apps/scraper` CLI | +| Category | What you get | Import | +| ------------------------------------------------------- | ------------------------------------------------------ | --------------------- | +| [Ballot & elections](#ballot--elections) | Contests, candidates, polling places, ballot measures | `@acme/api` | +| [Ballot-measure enrichment](#ballot-measure-enrichment) | Summaries, fiscal impact, pro/con arguments, citations | `@acme/api` | +| [State legislation](#state-legislation) | CA bills, legislators, votes | `@acme/api` | +| [Local government](#local-government) | City/county meetings, ordinances, votes | `@acme/api` | +| [CA election results](#ca-election-results) | Statewide vote counts (live JSON feed) | `@acme/api` | +| [Address autocomplete](#address-autocomplete) | Street-address suggestions + place details | `@acme/api` | +| [Corpus content (DB)](#corpus-content-db) | Bills, court cases, government content articles | tRPC `content` router | +| [Scrapers (background jobs)](#scrapers-background-jobs) | Congress bills, SCOTUS, Federal Register, LAO, CA SOS | `apps/scraper` CLI | --- @@ -32,26 +32,27 @@ All live-API functions come from `@acme/api`. Scrapers run as CLI jobs in `apps/ ```ts import { + getDistrictElectionResults, + getElectionResults, getElections, getVoterInfo, - getElectionResults, - getDistrictElectionResults, } from "@acme/api"; // Or, for direct tRPC consumption: // civic.getVoterInfo({ address, electionId }) ``` -| Function | Args | Returns | -|---|---|---| -| `getElections()` | — | `Election[]` — all elections Google knows about | -| `getVoterInfo(address, electionId)` | street address string, election ID | `VoterInfoResponse` — contests, polling places, drop boxes, enriched measures | -| `getElectionResults(stateFips, countyFips)` | FIPS codes | CA SOS results for the matching jurisdiction | -| `getDistrictElectionResults(districtRef)` | `DistrictRef` | narrowed by district | +| Function | Args | Returns | +| ------------------------------------------- | ---------------------------------- | ----------------------------------------------------------------------------- | +| `getElections()` | — | `Election[]` — all elections Google knows about | +| `getVoterInfo(address, electionId)` | street address string, election ID | `VoterInfoResponse` — contests, polling places, drop boxes, enriched measures | +| `getElectionResults(stateFips, countyFips)` | FIPS codes | CA SOS results for the matching jurisdiction | +| `getDistrictElectionResults(districtRef)` | `DistrictRef` | narrowed by district | **`getVoterInfo` runs the full enrichment pipeline** — ballot measures come back with summaries, fiscal impact, pro/con arguments, and per-field citations from the cross-validation engine. The raw Google Civic response alone omits nearly all content fields; this is what fills them. Key return types (all exported from `@acme/api`): + - `Contest` — a race or measure on the ballot (includes enriched `CanonicalMeasure` fields when present) - `Candidate` — name, party, photo, channels, incumbency, Vote Smart bio - `PollingLocation` — address, hours, type @@ -67,22 +68,21 @@ Key return types (all exported from `@acme/api`): **When to call directly:** when you need enriched measure data outside the `getVoterInfo` flow — e.g. article generation, batch enrichment scripts ```ts -import { - crossValidateMeasure, - type CanonicalMeasure, - type MeasureSourceData, - type MeasureCitation, - type SourceTier, - SOURCE_TIER_RANK, +import type { + CanonicalMeasure, + MeasureCitation, + MeasureSourceData, + SourceTier, } from "@acme/api"; +import type { CanonicalMeasure } from "@acme/api/lib/measure-sources/types"; +import { crossValidateMeasure, SOURCE_TIER_RANK } from "@acme/api"; // or via the named subpath: import { crossValidateMeasure } from "@acme/api/lib/measure-crossvalidate"; -import type { CanonicalMeasure } from "@acme/api/lib/measure-sources/types"; const result: CanonicalMeasure = await crossValidateMeasure( { title: "Proposition 1", - subtitle: "...", // optional — Google Civic's own text, used as last-resort + subtitle: "...", // optional — Google Civic's own text, used as last-resort text: "...", url: "...", }, @@ -96,32 +96,32 @@ const result: CanonicalMeasure = await crossValidateMeasure( `CanonicalMeasure` fields: -| Field | Type | Notes | -|---|---|---| -| `title` | `string` | Measure title (passed through) | -| `summary` | `string \| undefined` | Best available summary; cite `citations` for the source | -| `summaryShort` | `string \| undefined` | One-sentence card preview | -| `summaryLong` | `string \| undefined` | Fuller detail-screen paragraph | -| `summaryIsAiGenerated` | `boolean` | True when AI grounded on fetched text — UI should label it | -| `fiscalImpact` | `string \| undefined` | Official fiscal analysis (LAO for CA props, else Ballotpedia) | -| `fullText` | `string \| undefined` | Full measure text when available | -| `fullTextUrl` | `string \| undefined` | Link to the official text | -| `proArguments` | `MeasureArgument[]` | Attributed pro arguments | -| `conArguments` | `MeasureArgument[]` | Attributed con arguments | -| `citations` | `MeasureCitation[]` | One entry per populated field — `{field, sourceName, sourceUrl, tier, official}` | -| `discrepancies` | `string[] \| undefined` | Fields where top-2 sources disagreed; for human review | +| Field | Type | Notes | +| ---------------------- | ----------------------- | -------------------------------------------------------------------------------- | +| `title` | `string` | Measure title (passed through) | +| `summary` | `string \| undefined` | Best available summary; cite `citations` for the source | +| `summaryShort` | `string \| undefined` | One-sentence card preview | +| `summaryLong` | `string \| undefined` | Fuller detail-screen paragraph | +| `summaryIsAiGenerated` | `boolean` | True when AI grounded on fetched text — UI should label it | +| `fiscalImpact` | `string \| undefined` | Official fiscal analysis (LAO for CA props, else Ballotpedia) | +| `fullText` | `string \| undefined` | Full measure text when available | +| `fullTextUrl` | `string \| undefined` | Link to the official text | +| `proArguments` | `MeasureArgument[]` | Attributed pro arguments | +| `conArguments` | `MeasureArgument[]` | Attributed con arguments | +| `citations` | `MeasureCitation[]` | One entry per populated field — `{field, sourceName, sourceUrl, tier, official}` | +| `discrepancies` | `string[] \| undefined` | Fields where top-2 sources disagreed; for human review | **Sources wired into the engine** (in trust-tier order): -| Source | Tier | Scope | -|---|---|---| -| CA SOS Official Voter Guide | `state_sos` | CA statewide props — AG summary, official pro/con | -| CA LAO Fiscal Analyses | `state_sos` | CA statewide props — nonpartisan fiscal impact | -| LWV / CaVotes | `lwv` | CA statewide props — nonpartisan pros & cons | -| Ballotpedia | `ballotpedia` | Statewide + local lettered measures (all states) | -| Wikipedia | `wikipedia` | CA statewide props — encyclopedic overview | -| Vote Smart | `vote_smart` | State-level measures — summary + pro/con URLs | -| Google Civic | `google_civic` | The original input — lowest trust, always present | +| Source | Tier | Scope | +| --------------------------- | -------------- | ------------------------------------------------- | +| CA SOS Official Voter Guide | `state_sos` | CA statewide props — AG summary, official pro/con | +| CA LAO Fiscal Analyses | `state_sos` | CA statewide props — nonpartisan fiscal impact | +| LWV / CaVotes | `lwv` | CA statewide props — nonpartisan pros & cons | +| Ballotpedia | `ballotpedia` | Statewide + local lettered measures (all states) | +| Wikipedia | `wikipedia` | CA statewide props — encyclopedic overview | +| Vote Smart | `vote_smart` | State-level measures — summary + pro/con URLs | +| Google Civic | `google_civic` | The original input — lowest trust, always present | When no human source has a summary, the engine falls back to grounded AI (SPUR Bay Area voter guide as source text), flagged `summaryIsAiGenerated: true`. @@ -135,55 +135,80 @@ When no human source has a summary, the engine falls back to grounded AI (SPUR B **Scope:** California state bills and legislators (expandable to other states) ```ts +import type { + GetBillsOptions, + OpenStatesBill, + OpenStatesPerson, +} from "@acme/api"; import { - getBills, getBillDetails, - getLegislators, - getVotes, + getBills, + getBillsBySponsor, getCurrentSessions, getLegislatorById, - getBillsBySponsor, + getLegislators, + getVotes, openStatesClient, - type OpenStatesBill, - type OpenStatesPerson, - type GetBillsOptions, } from "@acme/api"; // or import { getBills } from "@acme/api/clients/open-states"; -const bills = await getBills({ state: "ca", session: "20232024", query: "housing" }); +const bills = await getBills({ + state: "ca", + session: "20232024", + query: "housing", +}); const detail = await getBillDetails("ocd-bill/..."); const legislators = await getLegislators({ state: "ca" }); ``` -| Function | What it returns | -|---|---| -| `getBills(opts)` | `OpenStatesBillSearchResult[]` — paginated bill search | -| `getBillDetails(billId)` | `OpenStatesBill` — full bill with actions, sponsors, versions | -| `getLegislators(opts)` | `OpenStatesPerson[]` — legislators matching the query | -| `getVotes(billId)` | `OpenStatesVote[]` — roll-call votes for a bill | -| `getCurrentSessions(state)` | Active legislative sessions | -| `getBillsBySponsor(personId)` | Bills sponsored by a legislator | -| `openStatesClient` | Raw client for custom queries | +| Function | What it returns | +| ----------------------------- | ------------------------------------------------------------- | +| `getBills(opts)` | `OpenStatesBillSearchResult[]` — paginated bill search | +| `getBillDetails(billId)` | `OpenStatesBill` — full bill with actions, sponsors, versions | +| `getLegislators(opts)` | `OpenStatesPerson[]` — legislators matching the query | +| `getVotes(billId)` | `OpenStatesVote[]` — roll-call votes for a bill | +| `getCurrentSessions(state)` | Active legislative sessions | +| `getBillsBySponsor(personId)` | Bills sponsored by a legislator | +| `openStatesClient` | Raw client for custom queries | --- ## Local government -**Source:** Legistar Web API -**Entry point:** `packages/api/src/integrations/legistar.ts` +**Sources:** persisted local-government records plus the Legistar Web API +**Entry points:** `packages/api/src/lib/local-government.ts`, `packages/api/src/integrations/legistar.ts` **Auth:** None (public API) -**Jurisdictions wired:** San Jose (`sanjose`), Santa Clara County (`santaclara`), Sunnyvale (`sunnyvale`) +**Persisted jurisdiction:** Cedar Park (`cedar-park-tx`); live Legistar jurisdictions: San Jose, Santa Clara County, Sunnyvale + +The product-facing reader uses normalized persisted records. Cedar Park's +scheduled scraper populates these from the city's official CivicEngage page and +embedded Municode Meetings publication: ```ts import { - legistar, - LegistarClient, - JURISDICTIONS, - type LegistarMeeting, - type LegistarMatter, - type LegistarVote, + getLocalGovernmentMeeting, + getLocalGovernmentMeetings, } from "@acme/api"; + +const meetings = await getLocalGovernmentMeetings({ + jurisdiction: "cedar-park-tx", + limit: 20, +}); +const detail = meetings[0] + ? await getLocalGovernmentMeeting(meetings[0].id) + : null; +``` + +The equivalent tRPC procedures are `localGovernment.meetings` and +`localGovernment.meeting`. Detail includes official/versioned documents, +agenda items, motions, outcomes, tally text, and named votes when published. + +Legistar remains available as a provider-specific live client: + +```ts +import type { LegistarMatter, LegistarMeeting, LegistarVote } from "@acme/api"; +import { JURISDICTIONS, legistar, LegistarClient } from "@acme/api"; // or import { legistar } from "@acme/api/integrations/legistar"; @@ -192,13 +217,13 @@ const matters = await legistar.getMatters("santaclara", { keywords: "budget" }); const votes = await legistar.getVotes("sanjose", matterId); ``` -| Method | Returns | -|---|---| -| `legistar.getMeetings(jurisdiction, opts?)` | `LegistarMeeting[]` — council meetings with agendas | -| `legistar.getMatters(jurisdiction, opts?)` | `LegistarMatter[]` — ordinances, resolutions, items | -| `legistar.getVotes(jurisdiction, matterId)` | `LegistarVote[]` — how each member voted | -| `legistar.getBodies(jurisdiction)` | `LegistarBody[]` — committees and boards | -| `legistar.getAttachments(jurisdiction, matterId)` | `LegistarAttachment[]` — PDFs, staff reports | +| Method | Returns | +| ------------------------------------------------- | --------------------------------------------------- | +| `legistar.getMeetings(jurisdiction, opts?)` | `LegistarMeeting[]` — council meetings with agendas | +| `legistar.getMatters(jurisdiction, opts?)` | `LegistarMatter[]` — ordinances, resolutions, items | +| `legistar.getVotes(jurisdiction, matterId)` | `LegistarVote[]` — how each member voted | +| `legistar.getBodies(jurisdiction)` | `LegistarBody[]` — committees and boards | +| `legistar.getAttachments(jurisdiction, matterId)` | `LegistarAttachment[]` — PDFs, staff reports | To add a city: add its `*.legistar.com` subdomain to `JURISDICTIONS` in `integrations/legistar.ts`. @@ -211,11 +236,8 @@ To add a city: add its `*.legistar.com` subdomain to `JURISDICTIONS` in `integra **Auth:** None ```ts -import { - SOS_RESULTS_HOME, - type ElectionContestResult, - type ResultCandidate, -} from "@acme/api"; +import type { ElectionContestResult, ResultCandidate } from "@acme/api"; +import { SOS_RESULTS_HOME } from "@acme/api"; // or import { SOS_RESULTS_HOME } from "@acme/api/clients/ca-sos-results"; ``` @@ -231,11 +253,8 @@ import { SOS_RESULTS_HOME } from "@acme/api/clients/ca-sos-results"; **Auth:** `GOOGLE_PLACES_API_KEY` (falls back through `GOOGLE_API_KEY` → `GOOGLE_CIVIC_API_KEY`) ```ts -import { - getAddressSuggestions, - getPlaceDetails, - type AddressSuggestion, -} from "@acme/api"; +import type { AddressSuggestion } from "@acme/api"; +import { getAddressSuggestions, getPlaceDetails } from "@acme/api"; const suggestions = await getAddressSuggestions("123 Main St"); const details = await getPlaceDetails(suggestions[0].placeId); @@ -261,24 +280,24 @@ const typed = await trpc.content.getByType.query({ type: "court_case" }); `ContentCard` shape (returned by all three procedures): -| Field | Type | -|---|---| -| `id` | `string` | -| `title` | `string` | -| `description` | `string \| null` | -| `type` | `"bill" \| "court_case" \| "government_content"` | -| `isAIGenerated` | `boolean` | -| `thumbnailUrl` | `string \| null` | +| Field | Type | +| --------------- | ------------------------------------------------ | +| `id` | `string` | +| `title` | `string` | +| `description` | `string \| null` | +| `type` | `"bill" \| "court_case" \| "government_content"` | +| `isAIGenerated` | `boolean` | +| `thumbnailUrl` | `string \| null` | `getThumbnailForContent(id, type)` is also exported from `@acme/api` for cases where only the thumbnail is needed. **DB tables populated by scrapers:** -| Table | Populated by | Content | -|---|---|---| -| `Bill` | `congress.ts` scraper | Federal bills, actions, sponsor, status, full text | -| `GovernmentContent` | `federalregister.ts` scraper | EOs, proclamations, presidential memos | -| `CourtCase` | `scotus.ts` scraper | SCOTUS opinions via CourtListener | +| Table | Populated by | Content | +| ------------------- | ---------------------------- | -------------------------------------------------- | +| `Bill` | `congress.ts` scraper | Federal bills, actions, sponsor, status, full text | +| `GovernmentContent` | `federalregister.ts` scraper | EOs, proclamations, presidential memos | +| `CourtCase` | `scotus.ts` scraper | SCOTUS opinions via CourtListener | --- @@ -286,15 +305,15 @@ const typed = await trpc.content.getByType.query({ type: "court_case" }); These run as CLI jobs (`bun run scrape -- --scrapers `) in `apps/scraper` and are **not importable as functions at request time**. They populate `CivicApiCache` rows or DB tables that the live API reads. -| Scraper | Populates | Cadence | -|---|---|---| -| `congress` | `Bill` table | Periodic | -| `federalregister` | `GovernmentContent` table | Periodic | -| `scotus` | `CourtCase` table | Periodic | -| `vote411` | `CivicApiCache` (VOTE411 voter guides) | Pre-election | -| `sccCvig` | `CivicApiCache` (SCC county voter guide) | Pre-election | -| `caSosStatements` | `CivicApiCache` (CA SOS candidate statements) | Pre-election | -| `caLaoFiscal` | `CivicApiCache` (LAO fiscal analyses) | Pre-election (~90 days out) | +| Scraper | Populates | Cadence | +| ----------------- | --------------------------------------------- | --------------------------- | +| `congress` | `Bill` table | Periodic | +| `federalregister` | `GovernmentContent` table | Periodic | +| `scotus` | `CourtCase` table | Periodic | +| `vote411` | `CivicApiCache` (VOTE411 voter guides) | Pre-election | +| `sccCvig` | `CivicApiCache` (SCC county voter guide) | Pre-election | +| `caSosStatements` | `CivicApiCache` (CA SOS candidate statements) | Pre-election | +| `caLaoFiscal` | `CivicApiCache` (LAO fiscal analyses) | Pre-election (~90 days out) | **How the cache-warmer pattern works:** a scraper pre-fetches expensive HTML pages and stores structured JSON in `CivicApiCache`. When a user triggers `getVoterInfo`, the measure-source adapters read the cache row (sub-millisecond DB read) instead of doing a live HTML fetch inside the request. Cache TTL is 30 days for LAO; 24 hours for voter info responses. diff --git a/docs/scraper.md b/docs/scraper.md index 68a83d3..2d23b29 100644 --- a/docs/scraper.md +++ b/docs/scraper.md @@ -15,21 +15,27 @@ process environment at runtime, not embedded during the build. ## Scrapers -| Scraper | Source | Content type | Method | -| ---------------------- | ------------------------------ | -------------------- | ------------------------------------------------------ | -| `congress.ts` | congress.gov REST API | `bill` | REST (`CONGRESS_API_KEY`), incremental by `updateDate` | -| `federalregister.ts` | federalregister.gov REST API | `government_content` | REST; HTML→Markdown via Turndown | -| `scotus.ts` | CourtListener REST API | `court_case` | REST (`COURTLISTENER_API_KEY`, optional) | -| `vote411.ts` | vote411.org | (cached locally) | cheerio HTML parse; does **not** write to the main DB | -| `scc-cvig.ts` | Santa Clara County voter guide | `civic_api_cache` | PDF extraction; optional Gemini fallback | -| `ca-sos-statements.ts` | CA Secretary of State guide | `civic_api_cache` | official candidate-statement pages | -| `ca-lao-fiscal.ts` | CA LAO ballot analyses | `civic_api_cache` | proposition fiscal analyses via HTML parse | -| `ca-vig-archive.ts` | CA SOS voter-guide archive | `civic_api_cache` | historical proposition guide pages via HTML parse | +| Scraper | Source | Content type | Method | +| ---------------------- | -------------------------------- | ----------------------- | ----------------------------------------------------------------------- | +| `congress.ts` | congress.gov REST API | `bill` | REST (`CONGRESS_API_KEY`), incremental by `updateDate` | +| `federalregister.ts` | federalregister.gov REST API | `government_content` | REST; HTML→Markdown via Turndown | +| `scotus.ts` | CourtListener REST API | `court_case` | REST (`COURTLISTENER_API_KEY`, optional) | +| `vote411.ts` | vote411.org | (cached locally) | cheerio HTML parse; does **not** write to the main DB | +| `scc-cvig.ts` | Santa Clara County voter guide | `civic_api_cache` | PDF extraction; optional Gemini fallback | +| `ca-sos-statements.ts` | CA Secretary of State guide | `civic_api_cache` | official candidate-statement pages | +| `civicengage.ts` | Cedar Park official council page | local-government tables | CivicEngage entry page + Municode embed; deterministic HTML/PDF parsing | +| `ca-lao-fiscal.ts` | CA LAO ballot analyses | `civic_api_cache` | proposition fiscal analyses via HTML parse | +| `ca-vig-archive.ts` | CA SOS voter-guide archive | `civic_api_cache` | historical proposition guide pages via HTML parse | All HTTP goes through one `fetchWithRetry()` utility (`apps/scraper/src/utils/fetch.ts`): exponential backoff (1s/2s/4s…), `Retry-After` support (seconds or HTTP-date), 30s default timeout via `AbortController`, retriable on 429/5xx and `ECONNRESET`/`ECONNREFUSED`, plus a stateful **per-host backoff** that ramps on 429/5xx and relaxes on success. > Note: `whitehouse.gov` cheerio scraping was replaced by the structured **Federal Register** REST API. `vote411-ballot.ts` exists for address-based ballot lookup (needs Playwright) but isn't wired into the CLI. +The Cedar Park pilot is deliberately limited to City Council meetings from the +latest 12 months. It stores meetings, versioned documents, agenda items, +motions/outcomes, and named roll-call votes in provider-neutral tables. It does +not browse or backfill historical election cycles. + ## Upsert + Change Detection `apps/scraper/src/utils/db/operations.ts` centralizes writes behind a discriminated-union `upsertContent(type, data)` (`type` ∈ bill | government_content | court_case). Each run: diff --git a/packages/api/package.json b/packages/api/package.json index 661eb54..3ff77ef 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -11,6 +11,10 @@ "types": "./dist/integrations/legistar.d.ts", "default": "./src/integrations/legistar.ts" }, + "./lib/local-government": { + "types": "./dist/lib/local-government.d.ts", + "default": "./src/lib/local-government.ts" + }, "./lib/civic": { "types": "./dist/lib/civic.d.ts", "default": "./src/lib/civic.ts" diff --git a/packages/api/src/index.ts b/packages/api/src/index.ts index 0d72933..70beef5 100644 --- a/packages/api/src/index.ts +++ b/packages/api/src/index.ts @@ -88,6 +88,13 @@ export { LegistarError, JURISDICTIONS, } from "./integrations/legistar"; + +// Provider-neutral local-government records persisted by source scrapers. +export { + getLocalGovernmentMeeting, + getLocalGovernmentMeetings, +} from "./lib/local-government"; +export type { LocalGovernmentMeetingQuery } from "./lib/local-government"; export type { Jurisdiction, LegistarMeeting, diff --git a/packages/api/src/lib/local-government.ts b/packages/api/src/lib/local-government.ts new file mode 100644 index 0000000..afbc1d8 --- /dev/null +++ b/packages/api/src/lib/local-government.ts @@ -0,0 +1,99 @@ +import { and, asc, desc, eq, gte, inArray, lte } from "@acme/db"; +import { db } from "@acme/db/client"; +import { + LocalGovernmentAgendaItem, + LocalGovernmentDocument, + LocalGovernmentMeeting, + LocalGovernmentVote, +} from "@acme/db/schema"; + +export interface LocalGovernmentMeetingQuery { + jurisdiction?: string; + start?: Date; + end?: Date; + limit?: number; +} + +export async function getLocalGovernmentMeetings( + query: LocalGovernmentMeetingQuery = {}, +) { + const filters = []; + if (query.jurisdiction) { + filters.push(eq(LocalGovernmentMeeting.jurisdiction, query.jurisdiction)); + } + if (query.start) + filters.push(gte(LocalGovernmentMeeting.startsAt, query.start)); + if (query.end) filters.push(lte(LocalGovernmentMeeting.startsAt, query.end)); + + return db + .select() + .from(LocalGovernmentMeeting) + .where(filters.length > 0 ? and(...filters) : undefined) + .orderBy(desc(LocalGovernmentMeeting.startsAt)) + .limit(Math.min(query.limit ?? 50, 100)); +} + +export async function getLocalGovernmentMeeting(id: string) { + const [meeting] = await db + .select() + .from(LocalGovernmentMeeting) + .where(eq(LocalGovernmentMeeting.id, id)) + .limit(1); + if (!meeting) return null; + + const [documents, items] = await Promise.all([ + db + .select({ + id: LocalGovernmentDocument.id, + meetingId: LocalGovernmentDocument.meetingId, + type: LocalGovernmentDocument.type, + title: LocalGovernmentDocument.title, + url: LocalGovernmentDocument.url, + mediaType: LocalGovernmentDocument.mediaType, + checksum: LocalGovernmentDocument.checksum, + isCurrent: LocalGovernmentDocument.isCurrent, + discoveredAt: LocalGovernmentDocument.discoveredAt, + fetchedAt: LocalGovernmentDocument.fetchedAt, + }) + .from(LocalGovernmentDocument) + .where(eq(LocalGovernmentDocument.meetingId, meeting.id)) + .orderBy( + desc(LocalGovernmentDocument.isCurrent), + asc(LocalGovernmentDocument.type), + ), + db + .select() + .from(LocalGovernmentAgendaItem) + .where(eq(LocalGovernmentAgendaItem.meetingId, meeting.id)) + .orderBy(asc(LocalGovernmentAgendaItem.sequence)), + ]); + + const votes = + items.length === 0 + ? [] + : await db + .select() + .from(LocalGovernmentVote) + .where( + inArray( + LocalGovernmentVote.agendaItemId, + items.map((item) => item.id), + ), + ) + .orderBy(asc(LocalGovernmentVote.voterName)); + const votesByItem = new Map(); + for (const vote of votes) { + const list = votesByItem.get(vote.agendaItemId) ?? []; + list.push(vote); + votesByItem.set(vote.agendaItemId, list); + } + + return { + ...meeting, + documents, + items: items.map((item) => ({ + ...item, + votes: votesByItem.get(item.id) ?? [], + })), + }; +} diff --git a/packages/api/src/root.ts b/packages/api/src/root.ts index ce0f415..c426f42 100644 --- a/packages/api/src/root.ts +++ b/packages/api/src/root.ts @@ -3,6 +3,7 @@ import { civicRouter } from "./router/civic"; import { contentRouter } from "./router/content"; import { feedbackRouter } from "./router/feedback"; import { legistarRouter } from "./router/legistar"; +import { localGovernmentRouter } from "./router/local-government"; import { openStatesRouter } from "./router/open-states"; import { placesRouter } from "./router/places"; import { postRouter } from "./router/post"; @@ -14,6 +15,7 @@ export const appRouter = createTRPCRouter({ auth: authRouter, civic: civicRouter, legistar: legistarRouter, + localGovernment: localGovernmentRouter, openStates: openStatesRouter, places: placesRouter, post: postRouter, diff --git a/packages/api/src/router/local-government.ts b/packages/api/src/router/local-government.ts new file mode 100644 index 0000000..996b083 --- /dev/null +++ b/packages/api/src/router/local-government.ts @@ -0,0 +1,44 @@ +import type { TRPCRouterRecord } from "@trpc/server"; +import { TRPCError } from "@trpc/server"; +import { z } from "zod/v4"; + +import { + getLocalGovernmentMeeting, + getLocalGovernmentMeetings, +} from "../lib/local-government"; +import { publicProcedure } from "../trpc"; + +export const localGovernmentRouter = { + meetings: publicProcedure + .input( + z + .object({ + jurisdiction: z.string().trim().min(1).default("cedar-park-tx"), + start: z.date().optional(), + end: z.date().optional(), + limit: z.number().int().min(1).max(100).default(50), + }) + .optional(), + ) + .query(({ input }) => + getLocalGovernmentMeetings({ + jurisdiction: input?.jurisdiction ?? "cedar-park-tx", + start: input?.start, + end: input?.end, + limit: input?.limit ?? 50, + }), + ), + + meeting: publicProcedure + .input(z.object({ id: z.uuid() })) + .query(async ({ input }) => { + const meeting = await getLocalGovernmentMeeting(input.id); + if (!meeting) { + throw new TRPCError({ + code: "NOT_FOUND", + message: "Meeting not found", + }); + } + return meeting; + }), +} satisfies TRPCRouterRecord; diff --git a/packages/db/migrations/add_local_government_tables.sql b/packages/db/migrations/add_local_government_tables.sql new file mode 100644 index 0000000..e65250c --- /dev/null +++ b/packages/db/migrations/add_local_government_tables.sql @@ -0,0 +1,77 @@ +CREATE TABLE IF NOT EXISTS "local_government_meeting" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL, + "source" varchar(50) NOT NULL, + "source_version" varchar(50) NOT NULL, + "jurisdiction" varchar(100) NOT NULL, + "governing_body" varchar(256) NOT NULL, + "external_id" varchar(128) NOT NULL, + "title" text NOT NULL, + "meeting_type" varchar(50) NOT NULL, + "status" varchar(50) NOT NULL, + "starts_at" timestamp with time zone NOT NULL, + "location" text, + "canonical_url" text NOT NULL, + "content_hash" varchar(64) NOT NULL, + "fetched_at" timestamp with time zone DEFAULT now() NOT NULL, + "created_at" timestamp DEFAULT now() NOT NULL, + "updated_at" timestamp with time zone, + CONSTRAINT "local_government_meeting_source_jurisdiction_external_id_unique" + UNIQUE("source", "jurisdiction", "external_id") +); +CREATE INDEX IF NOT EXISTS "local_government_meeting_jurisdiction_date_idx" + ON "local_government_meeting" ("jurisdiction", "starts_at"); + +CREATE TABLE IF NOT EXISTS "local_government_document" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL, + "meeting_id" uuid NOT NULL REFERENCES "local_government_meeting"("id") ON DELETE cascade, + "type" varchar(30) NOT NULL, + "title" text NOT NULL, + "url" text NOT NULL, + "media_type" varchar(100), + "checksum" varchar(64), + "extracted_text" text, + "is_current" boolean DEFAULT true NOT NULL, + "discovered_at" timestamp with time zone DEFAULT now() NOT NULL, + "fetched_at" timestamp with time zone, + "created_at" timestamp DEFAULT now() NOT NULL, + "updated_at" timestamp with time zone, + CONSTRAINT "local_government_document_meeting_id_type_url_unique" + UNIQUE("meeting_id", "type", "url") +); +CREATE INDEX IF NOT EXISTS "local_government_document_meeting_idx" + ON "local_government_document" ("meeting_id"); + +CREATE TABLE IF NOT EXISTS "local_government_agenda_item" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL, + "meeting_id" uuid NOT NULL REFERENCES "local_government_meeting"("id") ON DELETE cascade, + "external_id" varchar(128) NOT NULL, + "sequence" integer NOT NULL, + "item_number" varchar(50), + "section" varchar(100), + "item_type" varchar(50) NOT NULL, + "title" text NOT NULL, + "description" text, + "consent" boolean DEFAULT false NOT NULL, + "motion" text, + "outcome" varchar(100), + "vote_summary" text, + "source_url" text NOT NULL, + "created_at" timestamp DEFAULT now() NOT NULL, + "updated_at" timestamp with time zone, + CONSTRAINT "local_government_agenda_item_meeting_id_external_id_unique" + UNIQUE("meeting_id", "external_id") +); +CREATE INDEX IF NOT EXISTS "local_government_agenda_item_meeting_sequence_idx" + ON "local_government_agenda_item" ("meeting_id", "sequence"); + +CREATE TABLE IF NOT EXISTS "local_government_vote" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL, + "agenda_item_id" uuid NOT NULL REFERENCES "local_government_agenda_item"("id") ON DELETE cascade, + "voter_name" varchar(256) NOT NULL, + "value" varchar(50) NOT NULL, + "created_at" timestamp DEFAULT now() NOT NULL, + CONSTRAINT "local_government_vote_agenda_item_id_voter_name_unique" + UNIQUE("agenda_item_id", "voter_name") +); +CREATE INDEX IF NOT EXISTS "local_government_vote_agenda_item_idx" + ON "local_government_vote" ("agenda_item_id"); diff --git a/packages/db/src/schema.ts b/packages/db/src/schema.ts index 3c990ba..9587cb6 100644 --- a/packages/db/src/schema.ts +++ b/packages/db/src/schema.ts @@ -645,6 +645,132 @@ export const LegistarVote = pgTable( }), ); +// Provider-neutral local-government records populated by scheduled source +// adapters. Provider-specific tables above remain as the Legistar live cache; +// these tables are the durable contract consumed by the product. +export const LocalGovernmentMeeting = pgTable( + "local_government_meeting", + (t) => ({ + id: t.uuid().notNull().primaryKey().defaultRandom(), + source: t.varchar({ length: 50 }).notNull(), + sourceVersion: t.varchar({ length: 50 }).notNull(), + jurisdiction: t.varchar({ length: 100 }).notNull(), + governingBody: t.varchar({ length: 256 }).notNull(), + externalId: t.varchar({ length: 128 }).notNull(), + title: t.text().notNull(), + meetingType: t.varchar({ length: 50 }).notNull(), + status: t.varchar({ length: 50 }).notNull(), + startsAt: t.timestamp({ mode: "date", withTimezone: true }).notNull(), + location: t.text(), + canonicalUrl: t.text().notNull(), + contentHash: t.varchar({ length: 64 }).notNull(), + fetchedAt: t + .timestamp({ mode: "date", withTimezone: true }) + .defaultNow() + .notNull(), + createdAt: t.timestamp().defaultNow().notNull(), + updatedAt: t + .timestamp({ mode: "date", withTimezone: true }) + .$onUpdateFn(() => sql`now()`), + }), + (table) => ({ + uniqueSourceMeeting: unique().on( + table.source, + table.jurisdiction, + table.externalId, + ), + jurisdictionDateIdx: index( + "local_government_meeting_jurisdiction_date_idx", + ).on(table.jurisdiction, table.startsAt), + }), +); + +export const LocalGovernmentDocument = pgTable( + "local_government_document", + (t) => ({ + id: t.uuid().notNull().primaryKey().defaultRandom(), + meetingId: t + .uuid() + .notNull() + .references(() => LocalGovernmentMeeting.id, { onDelete: "cascade" }), + type: t.varchar({ length: 30 }).notNull(), + title: t.text().notNull(), + url: t.text().notNull(), + mediaType: t.varchar({ length: 100 }), + checksum: t.varchar({ length: 64 }), + extractedText: t.text(), + isCurrent: t.boolean().notNull().default(true), + discoveredAt: t + .timestamp({ mode: "date", withTimezone: true }) + .defaultNow() + .notNull(), + fetchedAt: t.timestamp({ mode: "date", withTimezone: true }), + createdAt: t.timestamp().defaultNow().notNull(), + updatedAt: t + .timestamp({ mode: "date", withTimezone: true }) + .$onUpdateFn(() => sql`now()`), + }), + (table) => ({ + uniqueMeetingDocument: unique().on(table.meetingId, table.type, table.url), + meetingDocumentIdx: index("local_government_document_meeting_idx").on( + table.meetingId, + ), + }), +); + +export const LocalGovernmentAgendaItem = pgTable( + "local_government_agenda_item", + (t) => ({ + id: t.uuid().notNull().primaryKey().defaultRandom(), + meetingId: t + .uuid() + .notNull() + .references(() => LocalGovernmentMeeting.id, { onDelete: "cascade" }), + externalId: t.varchar({ length: 128 }).notNull(), + sequence: t.integer().notNull(), + itemNumber: t.varchar({ length: 50 }), + section: t.varchar({ length: 100 }), + itemType: t.varchar({ length: 50 }).notNull(), + title: t.text().notNull(), + description: t.text(), + consent: t.boolean().notNull().default(false), + motion: t.text(), + outcome: t.varchar({ length: 100 }), + voteSummary: t.text(), + sourceUrl: t.text().notNull(), + createdAt: t.timestamp().defaultNow().notNull(), + updatedAt: t + .timestamp({ mode: "date", withTimezone: true }) + .$onUpdateFn(() => sql`now()`), + }), + (table) => ({ + uniqueMeetingItem: unique().on(table.meetingId, table.externalId), + meetingItemSequenceIdx: index( + "local_government_agenda_item_meeting_sequence_idx", + ).on(table.meetingId, table.sequence), + }), +); + +export const LocalGovernmentVote = pgTable( + "local_government_vote", + (t) => ({ + id: t.uuid().notNull().primaryKey().defaultRandom(), + agendaItemId: t + .uuid() + .notNull() + .references(() => LocalGovernmentAgendaItem.id, { onDelete: "cascade" }), + voterName: t.varchar({ length: 256 }).notNull(), + value: t.varchar({ length: 50 }).notNull(), + createdAt: t.timestamp().defaultNow().notNull(), + }), + (table) => ({ + uniqueItemVoter: unique().on(table.agendaItemId, table.voterName), + agendaItemVoteIdx: index("local_government_vote_agenda_item_idx").on( + table.agendaItemId, + ), + }), +); + // Google Civic API response cache export const CivicApiCache = pgTable( "civic_api_cache", diff --git a/packages/env/src/registry.ts b/packages/env/src/registry.ts index 6686f51..b5d7706 100644 --- a/packages/env/src/registry.ts +++ b/packages/env/src/registry.ts @@ -85,6 +85,11 @@ const scraperSourceLimitDefinitions = [ ["SCOTUS_MAX_ITEMS", "CourtListener opinion clusters per run.", "50"], ["SCC_CVIG_MAX_ITEMS", "Santa Clara voter-guide PDFs per run.", "10"], ["CA_SOS_MAX_ITEMS", "California SOS office pages per run.", "9"], + [ + "CEDAR_PARK_COUNCIL_MAX_ITEMS", + "Cedar Park City Council meetings per run.", + "100", + ], ] as const; export const envRegistry = [