diff --git a/apps/expo/src/components/UpcomingMeetingsSection.tsx b/apps/expo/src/components/UpcomingMeetingsSection.tsx index 93bbbf27..ef122c32 100644 --- a/apps/expo/src/components/UpcomingMeetingsSection.tsx +++ b/apps/expo/src/components/UpcomingMeetingsSection.tsx @@ -7,16 +7,16 @@ import { import { FontAwesome } from "@expo/vector-icons"; import { useQuery } from "@tanstack/react-query"; -import type { LegistarMeeting } from "@acme/api/integrations/legistar"; +import type { RouterOutputs } from "@acme/api"; import { Text, View } from "~/components/Themed"; import { fontBody, fontEditorial, fontSize, rd, sp, useTheme } from "~/styles"; import { trpc } from "~/utils/api"; +type Meeting = RouterOutputs["localGovernment"]["listMeetings"][number]; + interface UpcomingMeetingsSectionProps { - onMeetingPress?: ( - meeting: LegistarMeeting & { jurisdiction: string }, - ) => void; + onMeetingPress?: (meeting: Meeting) => void; } function formatDate(iso: string): string { @@ -34,7 +34,7 @@ export function UpcomingMeetingsSection({ const { theme } = useTheme(); const meetingsQuery = useQuery( - trpc.legistar.getMeetings.queryOptions({ daysAhead: 30 }), + trpc.localGovernment.listMeetings.queryOptions({ daysAhead: 90 }), ); return ( @@ -47,30 +47,43 @@ export function UpcomingMeetingsSection({ {meetingsQuery.data?.slice(0, 8).map((meeting, index) => ( onMeetingPress?.(meeting)} + onPress={() => + onMeetingPress + ? onMeetingPress(meeting) + : void Linking.openURL(meeting.canonicalUrl) + } activeOpacity={0.8} > {meeting.jurisdiction} - {formatDate(meeting.EventDate)} + + {formatDate(meeting.startsAt.toString())} + - {meeting.EventBodyName} + {meeting.isCancelled ? "Cancelled: " : ""} + {meeting.title} - {meeting.EventLocation && ( + {meeting.location && ( - {meeting.EventLocation} + {meeting.location} )} - {meeting.EventAgendaFile && ( + {meeting.documents.find( + (document) => document.type === "agenda", + ) && ( - void Linking.openURL(meeting.EventAgendaFile ?? "") + void Linking.openURL( + meeting.documents.find( + (document) => document.type === "agenda", + )?.url ?? "", + ) } hitSlop={8} > @@ -81,11 +94,9 @@ export function UpcomingMeetingsSection({ /> )} - {meeting.EventVideoPath && ( + {meeting.videoUrl && ( - void Linking.openURL(meeting.EventVideoPath ?? "") - } + onPress={() => void Linking.openURL(meeting.videoUrl ?? "")} hitSlop={8} > )} - {meeting.EventMinutesFile && ( + {meeting.documents.find( + (document) => document.type === "minutes", + ) && ( - void Linking.openURL(meeting.EventMinutesFile ?? "") + void Linking.openURL( + meeting.documents.find( + (document) => document.type === "minutes", + )?.url ?? "", + ) } hitSlop={8} > diff --git a/apps/scraper/README.md b/apps/scraper/README.md index 73793181..aca4ed62 100644 --- a/apps/scraper/README.md +++ b/apps/scraper/README.md @@ -4,15 +4,20 @@ Pulls in government content like bills, court cases, and White House content and ## Active data sources -Only these five are registered and run by `all`: - -| CLI name | Source and data fetched | Stored/used as | -| ------------------- | ------------------------------------------------------------------------------------------ | ----------------------------------------------------------------------------------------------------- | -| `federalregister` | Federal Register API presidential documents, then each document's body HTML | `government_content`; AI article/summary and feed-image enrichment | -| `congress` | Congress.gov API bill list, detail, CRS summaries, formatted text, and legislative actions | `bill`; powers federal bill content and AI/feed enrichment | -| `scotus` | CourtListener opinion clusters, dockets, and sub-opinion text for the Supreme Court | `court_case`; powers court content and AI/feed enrichment | -| `scc-cvig` | Hand-configured Santa Clara County voter-guide PDFs | Candidate statements in `CivicApiCache`; the API matches statements to candidates | -| `ca-sos-statements` | California SOS statewide-office candidate-statement pages | Candidate statements in `CivicApiCache`; the API reads the cache and can fall back to the live source | +These data sources are registered and run by `all`: + +| CLI name | Source and data fetched | Stored/used as | +| ------------------------ | ------------------------------------------------------------------------------------------ | ----------------------------------------------------------------------------------------------------- | +| `federalregister` | Federal Register API presidential documents, then each document's body HTML | `government_content`; AI article/summary and feed-image enrichment | +| `durham-bocc` | Durham County's official Legistar API (current election cycle only) | Provider-neutral meetings, agenda items, actions, votes, and official document links; no AI required | +| `congress` | Congress.gov API bill list, detail, CRS summaries, formatted text, and legislative actions | `bill`; powers federal bill content and AI/feed enrichment | +| `scotus` | CourtListener opinion clusters, dockets, and sub-opinion text for the Supreme Court | `court_case`; powers court content and AI/feed enrichment | +| `scc-cvig` | Hand-configured Santa Clara County voter-guide PDFs | Candidate statements in `CivicApiCache`; the API matches statements to candidates | +| `ca-sos-statements` | California SOS statewide-office candidate-statement pages | Candidate statements in `CivicApiCache`; the API reads the cache and can fall back to the live source | +| `ncsbe` | Current-cycle NCSBE candidate CSV, referendum PDFs, and result ZIPs | Provider-neutral election tables; powers `civic.getNcElectionData` with exact file provenance | +| `texas-legislature` | Texas Legislative Council anonymous FTP: current-session history XML and bulk documents | State-aware `bill` rows; read through `content.texasBills` and `content.getById` | +| `texas-current-election` | Texas SOS structured election feed and TLC amendment analyses | Current-cycle snapshots; powers `civic.getTexasCurrentElection` and measure enrichment | +| `cedar-park-council` | Cedar Park's CivicEngage City Council page and its official Municode Meetings embed | Provider-neutral local meetings, documents, agenda items, motions, outcomes, and votes | `vote411`, `ca-lao-fiscal`, and `ca-vig-archive` remain under `src/scrapers/disabled/` and do not run. Their caches had no application @@ -57,6 +62,7 @@ work: | `COURTLISTENER_API_KEY` | Optional | Higher CourtListener limits for `scotus`. | | `GOOGLE_API_KEY` / `GOOGLE_SEARCH_ENGINE_ID` | Optional pair | Google Custom Search article thumbnails. | | `GOOGLE_GENERATIVE_AI_API_KEY` | Optional | Gemini vision fallback for `scc-cvig` PDF extraction. | +| `OPEN_STATES_API_KEY` | Optional | Adds an exact Open States bill ID when Texas jurisdiction/session/identifier match. | See [the launch environment guide](../../docs/launch.md) for the complete per-scraper matrix, provider setup links, defaults, and production guidance. @@ -128,6 +134,11 @@ CONGRESS_MAX_ITEMS=10 pnpm --filter @acme/scraper run start congress | `SCOTUS_MAX_ITEMS` | 50 | CourtListener opinion clusters | | `SCC_CVIG_MAX_ITEMS` | 10 | Voter-guide PDF documents | | `CA_SOS_MAX_ITEMS` | 9 | Statewide-office candidate-statement pages | +| `NCSBE_MAX_ITEMS` | 4 | Current-cycle candidate/referendum/result files | +| `TEXAS_LEGISLATURE_MAX_ITEMS` | 100 | Bills from the latest Texas bulk session | +| `TX_SOS_MAX_ITEMS` | 12 | Current-cycle Texas SOS election payloads | +| `CEDAR_PARK_COUNCIL_MAX_ITEMS` | 100 | Council meetings (after the 12-month cutoff) | +| `DURHAM_BOCC_MAX_ITEMS` | 100 | Current-cycle Durham County BOCC meetings | | `SCRAPER_MAX_NEW_ITEMS_PER_RUN` | 10 | New records receiving expensive AI/image enrichment | These are per-run limits, not durable calendar-day quotas. Schedule one run per @@ -137,6 +148,33 @@ invocation gets a fresh allowance. Source limits cap API/page work; bills that require a generated description are deferred before insertion; other content may still be stored raw for later backfill. +The NCSBE integration is intentionally current-cycle only and excludes voter +history plus candidate contact/address fields. See +[`docs/ncsbe-election-data.md`](../../docs/ncsbe-election-data.md) for discovery, +idempotency, provenance, API, and deterministic Civic-matching details. + +## Cedar Park City Council (`civicengage.ts`) + +Cedar Park's public site is CivicEngage, but the City Council records page now +embeds a keyless Municode Meetings publish page. The adapter follows that +official embed, keeps a 12-month Council-only window, downloads at most two +documents concurrently, and deterministically parses PDF text. AI/vision is not +used. Agendas, packets, minutes, and later document URLs are versioned beneath +one provider-neutral meeting record; unchanged reruns produce the same natural +keys and checksums. + +```bash +pnpm --filter @acme/scraper run start cedar-park-council --max-items 1 +``` + +To add a second CivicEngage jurisdiction using the same kind of official +Municode embed, add a `CivicEngageJurisdictionConfig` beside +`cedarParkCouncilSource` with its CivicEngage host/path, IANA timezone, +Municode `cid`/`ppid`, and governing-body matcher, then instantiate the same +discovery/parser pipeline. If its records page uses Agenda Center, Legistar, or +another provider, implement that provider behind the same local-government +persistence contract instead. + --- ## Congress bills (`congress.ts`) @@ -157,6 +195,34 @@ await scrapeCongress({ --- +## Texas Legislature Online (`texas-legislature.ts`) + +Uses only the Texas Legislative Council's anonymous FTP service at +`ftp.legis.state.tx.us`; it does not crawl interactive TLO bill pages. The job: + +- discovers the newest session directory under `/bills` and rejects a stale + `TEXAS_LEGISLATURE_SESSION` assertion (official codes look like `89R` or + `892`); +- parses bill-history XML for identity, caption, sponsors, subjects, actions, + structured votes when present, and document metadata; +- downloads bill text, analyses, and fiscal notes from the matching FTP bulk + HTML paths and stores their extracted text alongside official HTML/PDF links; +- optionally stores an exact Open States ID without using Open States as the + legislative data source. + +Run a small current-session import: + +```bash +pnpm --filter @acme/scraper run start texas-legislature --max-items 10 +``` + +Apply `packages/db/migrations/add_state_legislation_fields.sql` before the first +run. The public `content.texasBills` procedure lists only the newest persisted +Texas session; `content.getById` returns its documents, actions, and votes. This +work deliberately does not provide historical-session browsing or backfills. + +--- + ## Court cases (`scotus.ts`) Uses the [CourtListener API](https://www.courtlistener.com/api/) — free, works without a key. Fetches recent opinions and pulls in the plain-text opinion content for AI article generation. @@ -184,6 +250,9 @@ All scrapers call into `src/utils/db/operations.ts`. Each time a bill or case is - If the **content changed** → regenerates the article - If **nothing changed** → backfills any missing AI summary/article/thumbnail fields, otherwise skips AI generation +The Texas importer passes `skipEnrichment` so official source data is persisted +without AI summaries, generated imagery, or videos. + Set `SCRAPER_FORCE_AI_REGEN=1` to force a full AI refresh even when the record already has AI content. For a new bill whose description must be generated, the summary is now created diff --git a/apps/scraper/package.json b/apps/scraper/package.json index c8a909f7..54832eac 100644 --- a/apps/scraper/package.json +++ b/apps/scraper/package.json @@ -13,6 +13,7 @@ "@ai-sdk/openai-compatible": "2.0.59", "@openrouter/ai-sdk-provider": "^2.10.0", "ai": "^6.0.141", + "basic-ftp": "^6.0.1", "cheerio": "^1.2.0", "consola": "^3.4.2", "drizzle-orm": "^0.45.2", diff --git a/apps/scraper/src/fixtures/ncsbe/candidates-2026.csv b/apps/scraper/src/fixtures/ncsbe/candidates-2026.csv new file mode 100644 index 00000000..cae1dfc3 --- /dev/null +++ b/apps/scraper/src/fixtures/ncsbe/candidates-2026.csv @@ -0,0 +1,3 @@ +"election_dt","county_name","contest_name","name_on_ballot","party_candidate","has_primary","is_partisan","vote_for","term" +"03/03/2026","DURHAM","US HOUSE OF REPRESENTATIVES DISTRICT 04","Nida Allam","DEM","TRUE","TRUE","1","2" +"03/03/2026","WAKE","WAKE COUNTY BOARD OF COMMISSIONERS AT-LARGE","Marguerite Creel","DEM","TRUE","TRUE","2","4" diff --git a/apps/scraper/src/fixtures/ncsbe/referendums-2026.txt b/apps/scraper/src/fixtures/ncsbe/referendums-2026.txt new file mode 100644 index 00000000..f3a816ea --- /dev/null +++ b/apps/scraper/src/fixtures/ncsbe/referendums-2026.txt @@ -0,0 +1,12 @@ +REFERENDUM CHOICES LIST GROUPED BY REFERENDUM +GATES BOARD OF ELECTIONS +CRITERIA: Election: 03/03/2026, County: ALL COUNTIES +GATES +GATES COUNTY LOCAL SALES AND USE TAX REFERENDUM +For Local sales and use tax at the rate of one-quarter percent (0.25%) in addition to all other State and local sales and use taxes. +Against Local sales and use tax at the rate of one-quarter percent (0.25%) in addition to all other State and local sales and use taxes. +GRANVILLE BOARD OF ELECTIONS +GRANVILLE +GRANVILLE COUNTY LOCAL SALES AND USE TAX REFERENDUM +For Local sales and use tax at the rate of one-quarter percent (0.25%) in addition to all other State and local sales and use taxes. +Against Local sales and use tax at the rate of one-quarter percent (0.25%) in addition to all other State and local sales and use taxes. diff --git a/apps/scraper/src/fixtures/ncsbe/results-2024.tsv b/apps/scraper/src/fixtures/ncsbe/results-2024.tsv new file mode 100644 index 00000000..3eb56e2c --- /dev/null +++ b/apps/scraper/src/fixtures/ncsbe/results-2024.tsv @@ -0,0 +1,3 @@ +county election_date precinct contest_group_id contest_type contest_name candidate party vote_for election_day early_voting absentee_by_mail provisional total_votes real_precinct +DURHAM 11/05/2024 01 1373 S US PRESIDENT Kamala D. Harris DEM 1 100 200 30 2 332 Y +WAKE 11/05/2024 01-01 1373 S US PRESIDENT Kamala D. Harris DEM 1 150 250 40 3 443 Y diff --git a/apps/scraper/src/fixtures/ncsbe/results-2026.tsv b/apps/scraper/src/fixtures/ncsbe/results-2026.tsv new file mode 100644 index 00000000..a08290a8 --- /dev/null +++ b/apps/scraper/src/fixtures/ncsbe/results-2026.tsv @@ -0,0 +1,3 @@ +County Election Date Precinct Contest Group ID Contest Type Contest Name Choice Choice Party Vote For Election Day Early Voting Absentee by Mail Provisional Total Votes Real Precinct +DURHAM 03/03/2026 04 2114 S US HOUSE OF REPRESENTATIVES DISTRICT 04 (DEM) Nida Allam DEM 1 371 0 0 0 371 Y +WAKE 03/03/2026 06-05 1 C WAKE COUNTY BOARD OF COMMISSIONERS AT-LARGE (DEM) Marguerite Creel DEM 2 12 0 0 0 12 Y diff --git a/apps/scraper/src/scraper-contracts.ts b/apps/scraper/src/scraper-contracts.ts index bfcfd04f..5e78aa8e 100644 --- a/apps/scraper/src/scraper-contracts.ts +++ b/apps/scraper/src/scraper-contracts.ts @@ -1,15 +1,27 @@ import type { ScraperEnvContract } from "@acme/env"; import { caSosStatementsConfig } from "./scrapers/ca-sos-statements.config.js"; +import { cedarParkCouncilConfig } from "./scrapers/civicengage.config.js"; import { congressConfig } from "./scrapers/congress.config.js"; +import { durhamBoccConfig } from "./scrapers/durham-bocc.config.js"; +import { durhamOnBaseConfig } from "./scrapers/durham-onbase.config.js"; import { federalregisterConfig } from "./scrapers/federalregister.config.js"; +import { ncsbeConfig } from "./scrapers/ncsbe.config.js"; import { sccCvigConfig } from "./scrapers/scc-cvig.config.js"; import { scotusConfig } from "./scrapers/scotus.config.js"; +import { texasCurrentElectionConfig } from "./scrapers/texas-current-election.config.js"; +import { texasLegislatureConfig } from "./scrapers/texas-legislature.config.js"; export const scraperContracts: readonly ScraperEnvContract[] = [ federalregisterConfig, + durhamBoccConfig, congressConfig, scotusConfig, sccCvigConfig, caSosStatementsConfig, + ncsbeConfig, + texasCurrentElectionConfig, + texasLegislatureConfig, + cedarParkCouncilConfig, + durhamOnBaseConfig, ]; diff --git a/apps/scraper/src/scrapers.ts b/apps/scraper/src/scrapers.ts index d4ab8d4d..2e3ad49f 100644 --- a/apps/scraper/src/scrapers.ts +++ b/apps/scraper/src/scrapers.ts @@ -1,14 +1,26 @@ import type { Scraper } from "./utils/types.js"; import { caSosStatements } from "./scrapers/ca-sos-statements.js"; +import { cedarParkCouncil } from "./scrapers/civicengage.js"; import { congress } from "./scrapers/congress.js"; +import { durhamBocc } from "./scrapers/durham-bocc.js"; +import { durhamOnBase } from "./scrapers/durham-onbase.js"; import { federalregister } from "./scrapers/federalregister.js"; +import { ncsbe } from "./scrapers/ncsbe.js"; import { sccCvig } from "./scrapers/scc-cvig.js"; import { scotus } from "./scrapers/scotus.js"; +import { texasCurrentElection } from "./scrapers/texas-current-election.js"; +import { texasLegislature } from "./scrapers/texas-legislature.js"; export const scrapers: readonly Scraper[] = [ federalregister, + durhamBocc, congress, scotus, sccCvig, caSosStatements, + ncsbe, + texasCurrentElection, + texasLegislature, + cedarParkCouncil, + durhamOnBase, ]; diff --git a/apps/scraper/src/scrapers/__fixtures__/durham-onbase-agenda.html b/apps/scraper/src/scrapers/__fixtures__/durham-onbase-agenda.html new file mode 100644 index 00000000..55a984b8 --- /dev/null +++ b/apps/scraper/src/scrapers/__fixtures__/durham-onbase-agenda.html @@ -0,0 +1,6 @@ + +
Consent Agenda
+
1. Approval of City Council Minutes
To approve the listed minutes. [Approved by Vote: 7/0]
+
General Business Agenda
+
21. Consolidated Annexation – 4802 Cheek Road
Motion 1: To adopt the ordinance. [FAILED by Vote: 0/7] Ayes: None. Nays: Mayor Williams, Council Member Baker. Motion 2: Consider rezoning. No vote taken.
+ diff --git a/apps/scraper/src/scrapers/__fixtures__/durham-onbase-index.html b/apps/scraper/src/scrapers/__fixtures__/durham-onbase-index.html new file mode 100644 index 00000000..ed7cac0d --- /dev/null +++ b/apps/scraper/src/scrapers/__fixtures__/durham-onbase-index.html @@ -0,0 +1,3 @@ + diff --git a/apps/scraper/src/scrapers/__fixtures__/durham-onbase-item.html b/apps/scraper/src/scrapers/__fixtures__/durham-onbase-item.html new file mode 100644 index 00000000..af308ff9 --- /dev/null +++ b/apps/scraper/src/scrapers/__fixtures__/durham-onbase-item.html @@ -0,0 +1,4 @@ +

Supporting Documents

+Final-Published Attachment - Approval Memo +March 16 City Council Minutes + diff --git a/apps/scraper/src/scrapers/civicengage-parser.test.ts b/apps/scraper/src/scrapers/civicengage-parser.test.ts new file mode 100644 index 00000000..3b87f7a6 --- /dev/null +++ b/apps/scraper/src/scrapers/civicengage-parser.test.ts @@ -0,0 +1,115 @@ +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import test from "node:test"; + +import { + parseAgendaItems, + parseCentralMeetingDate, + parseMunicodePublishPage, +} from "./civicengage-parser.js"; +import { cedarParkCouncilSource } from "./civicengage.config.js"; + +const fixture = (name: string) => + readFile(new URL(`./fixtures/civicengage/${name}`, import.meta.url), "utf8"); + +test("parses regular, special, cancelled, and amended meeting records", async () => { + const meetings = parseMunicodePublishPage( + await fixture("meetings.html"), + cedarParkCouncilSource, + { now: new Date("2026-03-01T00:00:00Z") }, + ); + assert.equal(meetings.length, 4); + assert.deepEqual( + meetings.map(({ title, meetingType, status }) => ({ + title, + meetingType, + status, + })), + [ + { + title: "City Council Mtg. - CANCELLED", + meetingType: "regular", + status: "cancelled", + }, + { + title: "City Council - Special Called", + meetingType: "special", + status: "held", + }, + { + title: "City Council Mtg.", + meetingType: "regular", + status: "completed", + }, + { + title: "City Council Mtg. - Amended", + meetingType: "regular", + status: "amended", + }, + ], + ); + assert.match(meetings[2]?.externalId ?? "", /^[a-f0-9]{32}$/); + assert.equal( + meetings[2]?.documents[0]?.url, + "https://mccmeetings.blob.core.usgovcloudapi.net/cptx-pubu/MEET-Agenda-2c1128fc619f48d6bf29cb94897a2d7f.pdf", + ); + assert.equal( + meetings[2]?.canonicalUrl, + "https://www.cedarparktexas.gov/596/City-Council-Agendas", + ); + assert.deepEqual( + meetings[3]?.documents.map(({ isCurrent }) => isCurrent), + [true, false], + ); +}); + +test("applies a twelve-month cutoff and parses Central daylight time", async () => { + const meetings = parseMunicodePublishPage( + await fixture("meetings.html"), + cedarParkCouncilSource, + { + now: new Date("2026-07-21T12:00:00Z"), + cutoff: new Date("2025-07-21T12:00:00Z"), + }, + ); + assert.equal(meetings.length, 3); + assert.equal( + parseCentralMeetingDate("2/12/2026", "6:00 PM").toISOString(), + "2026-02-13T00:00:00.000Z", + ); + assert.equal( + parseCentralMeetingDate("7/9/2026", "7:00 PM").toISOString(), + "2026-07-10T00:00:00.000Z", + ); +}); + +test("deterministically parses items, motions, outcomes, tallies, and roll calls", async () => { + const agenda = await fixture("agenda.txt"); + const minutes = await fixture("minutes.txt"); + const sourceUrl = "https://official.example/agenda.pdf"; + const first = parseAgendaItems(agenda, minutes, sourceUrl); + const second = parseAgendaItems(agenda, minutes, sourceUrl); + assert.deepEqual(first, second); + assert.equal(first.length, 6); + + const ordinance = first.find((item) => item.itemNumber === "E.1"); + assert.equal(ordinance?.itemType, "ordinance"); + assert.equal(ordinance?.consent, true); + assert.equal(ordinance?.outcome, "approved"); + + const resolution = first.find((item) => item.itemNumber === "F.1"); + assert.equal( + resolution?.motion, + "Motion to approve Agenda Item F.1 as presented.", + ); + assert.match(resolution?.voteSummary ?? "", /^6-0/); + assert.equal(resolution?.outcome, "approved"); + + const rollCall = first.find((item) => item.itemNumber === "H.1"); + assert.equal(rollCall?.votes.length, 6); + assert.deepEqual(rollCall?.votes.at(-1), { voterName: "Darby", value: "no" }); + assert.equal(rollCall?.sourceUrl, sourceUrl); + + const noAction = first.find((item) => item.itemNumber === "H.2"); + assert.equal(noAction?.outcome, "no_action"); +}); diff --git a/apps/scraper/src/scrapers/civicengage-parser.ts b/apps/scraper/src/scrapers/civicengage-parser.ts new file mode 100644 index 00000000..4f5760ed --- /dev/null +++ b/apps/scraper/src/scrapers/civicengage-parser.ts @@ -0,0 +1,414 @@ +import { createHash } from "node:crypto"; +import * as cheerio from "cheerio"; + +import type { CivicEngageJurisdictionConfig } from "./civicengage.config.js"; + +export type MeetingType = + | "regular" + | "special" + | "work_session" + | "retreat" + | "notice" + | "other"; +export type MeetingStatus = + | "scheduled" + | "held" + | "completed" + | "cancelled" + | "amended"; +export type DocumentType = "agenda" | "packet" | "minutes" | "html"; + +export interface DiscoveredDocument { + type: DocumentType; + title: string; + url: string; + mediaType: string; + isCurrent: boolean; +} + +export interface DiscoveredMeeting { + externalId: string; + title: string; + meetingType: MeetingType; + status: MeetingStatus; + startsAt: Date; + location?: string; + canonicalUrl: string; + documents: DiscoveredDocument[]; +} + +export interface ParsedVote { + voterName: string; + value: string; +} + +export interface ParsedAgendaItem { + externalId: string; + sequence: number; + itemNumber: string; + section?: string; + itemType: string; + title: string; + description?: string; + consent: boolean; + motion?: string; + outcome?: string; + voteSummary?: string; + sourceUrl: string; + votes: ParsedVote[]; +} + +const SOURCE_VERSION = "civicengage-municode-v1"; +export { SOURCE_VERSION }; + +function clean(value: string): string { + return value.replace(/\s+/g, " ").trim(); +} + +function directDocumentUrl(href: string, baseUrl: string): string { + const url = new URL(href, baseUrl); + if (url.hostname === "meetings.municode.com" && url.pathname === "/d/f") { + const target = url.searchParams.get("u"); + if (target) return new URL(target).toString(); + } + return url.toString(); +} + +function meetingType(title: string): MeetingType { + if (/notice\s+of\s+(?:a\s+)?possible\s+quorum/i.test(title)) return "notice"; + if (/retreat/i.test(title)) return "retreat"; + if (/work\s*shop|work\s+session/i.test(title)) return "work_session"; + if (/special\s+(?:called\s+)?(?:meeting|mtg)|special\s+called/i.test(title)) { + return "special"; + } + if (/council\s+(?:meeting|mtg)|regular/i.test(title)) return "regular"; + return "other"; +} + +function meetingStatus( + title: string, + startsAt: Date, + hasMinutes: boolean, + now: Date, +): MeetingStatus { + if (/cancel(?:led|ed|lation)/i.test(title)) return "cancelled"; + if (/amend(?:ed|ment)/i.test(title)) return "amended"; + if (hasMinutes) return "completed"; + return startsAt.getTime() > now.getTime() ? "scheduled" : "held"; +} + +export function parseMeetingDate( + dateText: string, + timeText: string, + timezone: string, +): Date { + const match = dateText.trim().match(/^(\d{1,2})\/(\d{1,2})\/(\d{4})$/); + const time = timeText.trim().match(/^(\d{1,2}):(\d{2})\s*([AP]M)$/i); + if (!match || !time) + throw new Error(`Unrecognized meeting date: ${dateText} ${timeText}`); + const month = Number(match[1]); + const day = Number(match[2]); + const year = Number(match[3]); + let hour = Number(time[1]) % 12; + if (time[3]?.toUpperCase() === "PM") hour += 12; + const minute = Number(time[2]); + const desiredUtc = Date.UTC(year, month - 1, day, hour, minute); + // Intl gives the local wall-clock parts for a UTC guess. The delta converts + // the desired local clock time to its real instant, including DST. + let instant = desiredUtc; + for (let pass = 0; pass < 2; pass++) { + const parts = new Intl.DateTimeFormat("en-US", { + timeZone: timezone, + year: "numeric", + month: "2-digit", + day: "2-digit", + hour: "2-digit", + minute: "2-digit", + hourCycle: "h23", + }).formatToParts(new Date(instant)); + const part = (type: Intl.DateTimeFormatPartTypes) => + Number(parts.find((value) => value.type === type)?.value); + const representedUtc = Date.UTC( + part("year"), + part("month") - 1, + part("day"), + part("hour"), + part("minute"), + ); + instant += desiredUtc - representedUtc; + } + return new Date(instant); +} + +export function parseCentralMeetingDate( + dateText: string, + timeText: string, +): Date { + return parseMeetingDate(dateText, timeText, "America/Chicago"); +} + +function stableMeetingId(title: string, startsAt: Date): string { + const normalizedTitle = title + .replace(/\b(?:amended|cancelled|canceled)\b/gi, "") + .replace(/[^a-z0-9]+/gi, " ") + .trim() + .toLowerCase(); + return createHash("sha256") + .update(`${startsAt.toISOString()}|${normalizedTitle}`) + .digest("hex") + .slice(0, 32); +} + +export function parseMunicodePublishPage( + html: string, + config: CivicEngageJurisdictionConfig, + options: { now?: Date; cutoff?: Date } = {}, +): DiscoveredMeeting[] { + const $ = cheerio.load(html); + const now = options.now ?? new Date(); + const canonicalUrl = new URL( + config.recordsPagePath, + config.civicEngageBaseUrl, + ).toString(); + const meetings: DiscoveredMeeting[] = []; + + $("table tr").each((_index, row) => { + const $row = $(row); + const title = clean($row.find("td.meeting").text()); + const dateText = clean($row.find("td.date").text()); + const timeText = clean($row.find("td.time").text()); + if ( + !title || + !dateText || + !timeText || + !config.provider.meetingNamePattern.test(title) + ) + return; + + let startsAt: Date; + try { + startsAt = parseMeetingDate(dateText, timeText, config.timezone); + } catch { + return; + } + if (options.cutoff && startsAt < options.cutoff) return; + + const documents: DiscoveredDocument[] = []; + for (const type of ["agenda", "packet", "minutes"] as const) { + const link = $row.find(`td.${type} a`).first(); + const href = link.attr("href"); + if (!href) continue; + documents.push({ + type, + title: clean(link.find("img").attr("alt") ?? `${type} for ${title}`), + url: directDocumentUrl(href, "https://meetings.municode.com"), + mediaType: "application/pdf", + isCurrent: true, + }); + } + + meetings.push({ + externalId: stableMeetingId(title, startsAt), + title, + meetingType: meetingType(title), + status: meetingStatus( + title, + startsAt, + documents.some((document) => document.type === "minutes"), + now, + ), + startsAt, + location: clean($row.find("td.venue").text()) || undefined, + canonicalUrl, + documents, + }); + }); + + const grouped = new Map(); + for (const candidate of meetings) { + const existing = grouped.get(candidate.externalId); + if (!existing) { + grouped.set(candidate.externalId, candidate); + continue; + } + const candidatePreferred = + ["amended", "cancelled"].includes(candidate.status) && + !["amended", "cancelled"].includes(existing.status); + const preferred = candidatePreferred ? candidate : existing; + const secondary = candidatePreferred ? existing : candidate; + const preferredTypes = new Set( + preferred.documents.map((document) => document.type), + ); + const documents = [ + ...preferred.documents.map((document) => ({ + ...document, + isCurrent: true, + })), + ...secondary.documents.map((document) => ({ + ...document, + isCurrent: !preferredTypes.has(document.type), + })), + ].filter( + (document, index, all) => + all.findIndex( + (candidateDocument) => candidateDocument.url === document.url, + ) === index, + ); + grouped.set(candidate.externalId, { ...preferred, documents }); + } + + return [...grouped.values()].sort( + (a, b) => b.startsAt.getTime() - a.startsAt.getTime(), + ); +} + +const SECTION_NAMES = [ + "Consent Agenda", + "Public Hearings", + "Regular Agenda (Non-Consent)", + "Regular Agenda", + "Executive Session", + "Open Meeting", +]; + +function normalizedLines(text: string): string[] { + return text + .replace(/\r/g, "") + .split("\n") + .map((line) => clean(line)) + .filter(Boolean) + .filter((line) => !/^City Council (?:Agenda|Minutes)$/i.test(line)) + .filter( + (line) => + !/^(?:January|February|March|April|May|June|July|August|September|October|November|December) \d{1,2}, \d{4}$/i.test( + line, + ), + ); +} + +interface TextItemBlock { + number: string; + text: string; + section?: string; +} + +function itemBlocks(text: string): TextItemBlock[] { + const lines = normalizedLines(text); + const blocks: TextItemBlock[] = []; + let section: string | undefined; + let current: TextItemBlock | undefined; + + for (const line of lines) { + const sectionName = SECTION_NAMES.find( + (candidate) => candidate.toLowerCase() === line.toLowerCase(), + ); + if (sectionName) { + section = sectionName; + continue; + } + const start = line.match(/^([A-Z]{1,2}\.\d+)\s+(.+)$/); + if (start) { + if (current) blocks.push(current); + current = { number: start[1]!, text: start[2]!, section }; + } else if (current && !/^(?:Page \d+|-{3,})$/i.test(line)) { + current.text += `\n${line}`; + } + } + if (current) blocks.push(current); + return blocks; +} + +function itemType(text: string, section?: string): string { + if (/public hearing/i.test(text) || /public hearings?/i.test(section ?? "")) + return "public_hearing"; + if (/\bordinance\b/i.test(text)) return "ordinance"; + if (/\bresolution\b/i.test(text)) return "resolution"; + if (/\bminutes\b/i.test(text)) return "minutes"; + if (/presentation|proclamation/i.test(text)) return "presentation"; + if (/executive session/i.test(section ?? "")) return "executive_session"; + return "agenda_item"; +} + +function firstMatch(text: string, pattern: RegExp): string | undefined { + return clean(text.match(pattern)?.[1] ?? "") || undefined; +} + +function parseRollCall(text: string): ParsedVote[] { + const votes: ParsedVote[] = []; + const patterns: [RegExp, string][] = [ + [/(?:Voting|Voted)\s+(?:Aye|Yes):\s*([^\n]+)/gi, "yes"], + [/(?:Voting|Voted)\s+(?:Nay|No):\s*([^\n]+)/gi, "no"], + [/Abstain(?:ing)?:\s*([^\n]+)/gi, "abstain"], + [/Absent:\s*([^\n]+)/gi, "absent"], + ]; + for (const [pattern, value] of patterns) { + let match: RegExpExecArray | null; + while ((match = pattern.exec(text)) !== null) { + for (const rawName of (match[1] ?? "").split(/,|\band\b/i)) { + const voterName = clean(rawName.replace(/Council(?:member)?/gi, "")); + if (voterName) votes.push({ voterName, value }); + } + } + } + return votes; +} + +function outcome(text: string, voteSummary?: string): string | undefined { + if (/no action taken/i.test(text)) return "no_action"; + if (/approved under the consent agenda|motion (?:carried|passed)/i.test(text)) + return "approved"; + const tally = voteSummary?.match(/(\d+)\s*[-–]\s*(\d+)/); + if (tally) return Number(tally[1]) > Number(tally[2]) ? "approved" : "failed"; + return undefined; +} + +export function parseAgendaItems( + agendaText: string, + minutesText: string | undefined, + sourceUrl: string, +): ParsedAgendaItem[] { + const agenda = itemBlocks(agendaText); + const minutes = new Map( + itemBlocks(minutesText ?? "").map((block) => [block.number, block]), + ); + + return agenda.map((block, index) => { + const minutesBlock = minutes.get(block.number); + const combined = minutesBlock?.text ?? block.text; + const parsedMotion = firstMatch( + combined, + /(Motion to[\s\S]*?)(?=\n(?:Movant|Second|Vote):|$)/i, + ); + const motion = + parsedMotion && + /Agenda Items?/i.test(parsedMotion) && + !new RegExp(`\\b${block.number.replace(".", "\\.")}\\b`, "i").test( + parsedMotion, + ) && + !/consent agenda/i.test(block.section ?? "") + ? undefined + : parsedMotion; + const voteSummary = firstMatch(combined, /Vote:\s*([^\n]+)/i); + const title = clean(block.text.split("\n")[0] ?? block.text); + const description = clean( + block.text.replace(block.text.split("\n")[0] ?? "", ""), + ); + return { + externalId: block.number.toLowerCase(), + sequence: index + 1, + itemNumber: block.number, + section: block.section, + itemType: itemType(block.text, block.section), + title, + description: description || undefined, + consent: + /consent agenda/i.test(block.section ?? "") || + /approved under the consent agenda/i.test(combined), + motion, + outcome: outcome(combined, voteSummary), + voteSummary, + sourceUrl, + votes: parseRollCall(combined), + }; + }); +} diff --git a/apps/scraper/src/scrapers/civicengage.config.ts b/apps/scraper/src/scrapers/civicengage.config.ts new file mode 100644 index 00000000..0bb5befc --- /dev/null +++ b/apps/scraper/src/scrapers/civicengage.config.ts @@ -0,0 +1,57 @@ +import type { ScraperEnvContract } from "@acme/env"; + +export interface CivicEngageJurisdictionConfig { + id: string; + name: string; + governingBody: string; + civicEngageBaseUrl: string; + recordsPagePath: string; + timezone: string; + provider: { + kind: "municode-publish-page"; + clientId: string; + publishPageId: string; + meetingNamePattern: RegExp; + }; +} + +/** + * Cedar Park's public website is CivicEngage, but its City Council records page + * now embeds Municode Meetings. Keeping both halves in configuration makes the + * adapter reusable when another CivicEngage jurisdiction uses the same embed, + * without pretending all CivicEngage Agenda Centers share this provider. + */ +export const cedarParkCouncilSource: CivicEngageJurisdictionConfig = { + id: "cedar-park-tx", + name: "City of Cedar Park, Texas", + governingBody: "City Council", + civicEngageBaseUrl: "https://www.cedarparktexas.gov", + recordsPagePath: "/596/City-Council-Agendas", + timezone: "America/Chicago", + provider: { + kind: "municode-publish-page", + clientId: "CPTX", + publishPageId: "d5927f56-2e55-4a02-a095-4ed5e6109cfd", + meetingNamePattern: /\b(?:city\s+)?council\b/i, + }, +}; + +export function municodePublishPageUrl( + config: CivicEngageJurisdictionConfig, +): string { + const url = new URL("https://meetings.municode.com/PublishPage/index"); + url.searchParams.set("cid", config.provider.clientId); + url.searchParams.set("ppid", config.provider.publishPageId); + url.searchParams.set("p", "-1"); + return url.toString(); +} + +export const cedarParkCouncilConfig = { + id: "cedar-park-council", + name: "Cedar Park City Council", + source: "Cedar Park CivicEngage / official Municode Meetings embed", + environment: { + required: ["POSTGRES_URL"], + optional: ["CEDAR_PARK_COUNCIL_MAX_ITEMS"], + }, +} as const satisfies ScraperEnvContract; diff --git a/apps/scraper/src/scrapers/civicengage.ts b/apps/scraper/src/scrapers/civicengage.ts new file mode 100644 index 00000000..dec3f12a --- /dev/null +++ b/apps/scraper/src/scrapers/civicengage.ts @@ -0,0 +1,353 @@ +import { createHash } from "node:crypto"; +import pLimit from "p-limit"; +import { getDocumentProxy } from "unpdf"; + +import { and, eq, notInArray } from "@acme/db"; +import { db } from "@acme/db/client"; +import { + LocalGovernmentAgendaItem, + LocalGovernmentDocument, + LocalGovernmentMeeting, + LocalGovernmentVote, +} from "@acme/db/schema"; + +import type { Scraper } from "../utils/types.js"; +import type { + DiscoveredDocument, + DiscoveredMeeting, + ParsedAgendaItem, +} from "./civicengage-parser.js"; +import { setExpectedTotal } from "../utils/db/metrics.js"; +import { fetchWithRetry } from "../utils/fetch.js"; +import { createLogger } from "../utils/log.js"; +import { + parseAgendaItems, + parseMunicodePublishPage, + SOURCE_VERSION, +} from "./civicengage-parser.js"; +import { + cedarParkCouncilConfig, + cedarParkCouncilSource, + municodePublishPageUrl, +} from "./civicengage.config.js"; + +const logger = createLogger("cedar-park-council"); +const documentLimit = pLimit(2); +const USER_AGENT = + "Billion civic data scraper (+https://github.com/billion-app/billion)"; + +interface FetchedDocument extends DiscoveredDocument { + checksum?: string; + extractedText?: string; + fetchedAt?: Date; +} + +function checksum(bytes: Uint8Array): string { + return createHash("sha256").update(bytes).digest("hex"); +} + +export async function extractPdfText(bytes: Uint8Array): Promise { + const proxy = await getDocumentProxy(bytes); + const pages: string[] = []; + for (let pageNumber = 1; pageNumber <= proxy.numPages; pageNumber++) { + const page = await proxy.getPage(pageNumber); + const content = await page.getTextContent(); + const items = content.items + .map((raw) => { + const item = raw as { str?: string; transform?: number[] }; + return item.str && item.transform + ? { + text: item.str, + x: item.transform[4] ?? 0, + y: item.transform[5] ?? 0, + } + : null; + }) + .filter( + (item): item is { text: string; x: number; y: number } => item !== null, + ) + .sort((a, b) => (Math.abs(a.y - b.y) > 2 ? b.y - a.y : a.x - b.x)); + + const lines: { y: number; parts: string[] }[] = []; + for (const item of items) { + const line = lines.find( + (candidate) => Math.abs(candidate.y - item.y) <= 2, + ); + if (line) line.parts.push(item.text); + else lines.push({ y: item.y, parts: [item.text] }); + } + pages.push(lines.map((line) => line.parts.join(" ")).join("\n")); + } + return pages.join("\n\n"); +} + +async function fetchDocument( + document: DiscoveredDocument, +): Promise { + const [cached] = await db + .select({ + checksum: LocalGovernmentDocument.checksum, + extractedText: LocalGovernmentDocument.extractedText, + fetchedAt: LocalGovernmentDocument.fetchedAt, + }) + .from(LocalGovernmentDocument) + .where(eq(LocalGovernmentDocument.url, document.url)) + .limit(1); + if (cached?.checksum) { + return { + ...document, + checksum: cached.checksum, + extractedText: cached.extractedText ?? undefined, + fetchedAt: cached.fetchedAt ?? undefined, + }; + } + + try { + const response = await fetchWithRetry(document.url, { + headers: { Accept: document.mediaType, "User-Agent": USER_AGENT }, + timeoutMs: 60_000, + }); + const bytes = new Uint8Array(await response.arrayBuffer()); + const shouldExtract = + document.type === "agenda" || document.type === "minutes"; + return { + ...document, + checksum: checksum(bytes), + extractedText: shouldExtract ? await extractPdfText(bytes) : undefined, + fetchedAt: new Date(), + }; + } catch (error) { + logger.warn(`Could not fetch ${document.type} ${document.url}`, error); + return document; + } +} + +function meetingHash( + meeting: DiscoveredMeeting, + documents: FetchedDocument[], + items: ParsedAgendaItem[], +): string { + return createHash("sha256") + .update( + JSON.stringify({ + title: meeting.title, + meetingType: meeting.meetingType, + status: meeting.status, + startsAt: meeting.startsAt.toISOString(), + location: meeting.location, + documents: documents.map(({ type, url, checksum, isCurrent }) => ({ + type, + url, + checksum, + isCurrent, + })), + items, + }), + ) + .digest("hex"); +} + +async function persistMeeting( + meeting: DiscoveredMeeting, + documents: FetchedDocument[], + items: ParsedAgendaItem[], +): Promise { + const fetchedAt = new Date(); + const contentHash = meetingHash(meeting, documents, items); + + await db.transaction(async (tx) => { + const [row] = await tx + .insert(LocalGovernmentMeeting) + .values({ + source: "civicengage", + sourceVersion: SOURCE_VERSION, + jurisdiction: cedarParkCouncilSource.id, + governingBody: cedarParkCouncilSource.governingBody, + externalId: meeting.externalId, + title: meeting.title, + meetingType: meeting.meetingType, + status: meeting.status, + startsAt: meeting.startsAt, + location: meeting.location, + canonicalUrl: meeting.canonicalUrl, + contentHash, + fetchedAt, + }) + .onConflictDoUpdate({ + target: [ + LocalGovernmentMeeting.source, + LocalGovernmentMeeting.jurisdiction, + LocalGovernmentMeeting.externalId, + ], + set: { + sourceVersion: SOURCE_VERSION, + title: meeting.title, + meetingType: meeting.meetingType, + status: meeting.status, + startsAt: meeting.startsAt, + location: meeting.location, + canonicalUrl: meeting.canonicalUrl, + contentHash, + fetchedAt, + updatedAt: fetchedAt, + }, + }) + .returning({ id: LocalGovernmentMeeting.id }); + if (!row) + throw new Error(`Failed to persist meeting ${meeting.externalId}`); + + await tx + .update(LocalGovernmentDocument) + .set({ isCurrent: false, updatedAt: fetchedAt }) + .where(eq(LocalGovernmentDocument.meetingId, row.id)); + + for (const document of documents) { + await tx + .insert(LocalGovernmentDocument) + .values({ + meetingId: row.id, + type: document.type, + title: document.title, + url: document.url, + mediaType: document.mediaType, + checksum: document.checksum, + extractedText: document.extractedText, + isCurrent: document.isCurrent, + fetchedAt: document.fetchedAt, + }) + .onConflictDoUpdate({ + target: [ + LocalGovernmentDocument.meetingId, + LocalGovernmentDocument.type, + LocalGovernmentDocument.url, + ], + set: { + title: document.title, + mediaType: document.mediaType, + checksum: document.checksum, + extractedText: document.extractedText, + isCurrent: document.isCurrent, + fetchedAt: document.fetchedAt, + updatedAt: fetchedAt, + }, + }); + } + + for (const item of items) { + const [itemRow] = await tx + .insert(LocalGovernmentAgendaItem) + .values({ + meetingId: row.id, + externalId: item.externalId, + sequence: item.sequence, + itemNumber: item.itemNumber, + section: item.section, + itemType: item.itemType, + title: item.title, + description: item.description, + consent: item.consent, + motion: item.motion, + outcome: item.outcome, + voteSummary: item.voteSummary, + sourceUrl: item.sourceUrl, + }) + .onConflictDoUpdate({ + target: [ + LocalGovernmentAgendaItem.meetingId, + LocalGovernmentAgendaItem.externalId, + ], + set: { + sequence: item.sequence, + itemNumber: item.itemNumber, + section: item.section, + itemType: item.itemType, + title: item.title, + description: item.description, + consent: item.consent, + motion: item.motion, + outcome: item.outcome, + voteSummary: item.voteSummary, + sourceUrl: item.sourceUrl, + updatedAt: fetchedAt, + }, + }) + .returning({ id: LocalGovernmentAgendaItem.id }); + if (!itemRow) continue; + await tx + .delete(LocalGovernmentVote) + .where(eq(LocalGovernmentVote.agendaItemId, itemRow.id)); + if (item.votes.length === 0) continue; + await tx.insert(LocalGovernmentVote).values( + item.votes.map((vote) => ({ + agendaItemId: itemRow.id, + voterName: vote.voterName, + value: vote.value, + })), + ); + } + const staleItemFilter = + items.length === 0 + ? eq(LocalGovernmentAgendaItem.meetingId, row.id) + : and( + eq(LocalGovernmentAgendaItem.meetingId, row.id), + notInArray( + LocalGovernmentAgendaItem.externalId, + items.map((item) => item.externalId), + ), + ); + await tx.delete(LocalGovernmentAgendaItem).where(staleItemFilter); + }); +} + +async function processMeeting(meeting: DiscoveredMeeting): Promise { + const documents = await Promise.all( + meeting.documents.map((document) => + documentLimit(() => fetchDocument(document)), + ), + ); + const agenda = documents.find((document) => document.type === "agenda"); + const minutes = documents.find((document) => document.type === "minutes"); + const items = agenda?.extractedText + ? parseAgendaItems(agenda.extractedText, minutes?.extractedText, agenda.url) + : []; + await persistMeeting(meeting, documents, items); + logger.success( + `${meeting.startsAt.toISOString().slice(0, 10)} ${meeting.title}: ${items.length} items`, + ); +} + +async function scrape(maxItems = 100): Promise { + const now = new Date(); + const cutoff = new Date(now); + cutoff.setUTCFullYear(cutoff.getUTCFullYear() - 1); + const listingUrl = municodePublishPageUrl(cedarParkCouncilSource); + logger.info( + `Discovering Cedar Park Council meetings since ${cutoff.toISOString()}`, + ); + const response = await fetchWithRetry(listingUrl, { + headers: { Accept: "text/html", "User-Agent": USER_AGENT }, + }); + const meetings = parseMunicodePublishPage( + await response.text(), + cedarParkCouncilSource, + { + now, + cutoff, + }, + ).slice(0, maxItems); + setExpectedTotal(meetings.length); + + // Meetings are intentionally sequential; only document requests within one + // meeting run concurrently, capped at two against the public records host. + for (const meeting of meetings) await processMeeting(meeting); + logger.success(`Completed ${meetings.length} Cedar Park Council meetings`); +} + +export const cedarParkCouncil: Scraper = { + ...cedarParkCouncilConfig, + scrape: (options) => + scrape( + (options?.maxItems ?? Number(process.env.CEDAR_PARK_COUNCIL_MAX_ITEMS)) || + 100, + ), +}; diff --git a/apps/scraper/src/scrapers/durham-bocc.config.ts b/apps/scraper/src/scrapers/durham-bocc.config.ts new file mode 100644 index 00000000..882a29a2 --- /dev/null +++ b/apps/scraper/src/scrapers/durham-bocc.config.ts @@ -0,0 +1,11 @@ +import type { ScraperEnvContract } from "@acme/env"; + +export const durhamBoccConfig = { + id: "durham-bocc", + name: "Durham County BOCC", + source: "Durham County Legistar Web API — BOCC meetings and actions", + environment: { + required: ["POSTGRES_URL"], + optional: ["DURHAM_BOCC_MAX_ITEMS"], + }, +} as const satisfies ScraperEnvContract; diff --git a/apps/scraper/src/scrapers/durham-bocc.test.ts b/apps/scraper/src/scrapers/durham-bocc.test.ts new file mode 100644 index 00000000..aab5679a --- /dev/null +++ b/apps/scraper/src/scrapers/durham-bocc.test.ts @@ -0,0 +1,74 @@ +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import test from "node:test"; + +import { + adaptDurhamItem, + adaptDurhamMeeting, + adaptDurhamVote, + currentElectionCycleStart, + parseDurhamStart, +} from "./durham-bocc.js"; + +async function fixture(name: string): Promise { + const url = new URL(`./fixtures/durham-bocc/${name}.json`, import.meta.url); + return JSON.parse(await readFile(url, "utf8")) as unknown; +} + +void test("maps regular, work, cancelled, and amended meetings", async () => { + const events = (await fixture("events")) as unknown[]; + const [regular, work, cancelled, amended] = events.map(adaptDurhamMeeting); + + assert.equal(regular?.meetingType, "Regular Session"); + assert.equal(regular?.isCancelled, false); + assert.match(regular?.videoUrl ?? "", /ID1=1539/); + assert.equal(work?.meetingType, "Work Session"); + assert.equal(cancelled?.isCancelled, true); + assert.equal(cancelled?.status, "cancelled"); + assert.equal(amended?.isAmended, true); + assert.equal(amended?.documents[0]?.title, "Amended Agenda"); +}); + +void test("maps actions, consent status, official attachments, and named votes", async () => { + const [rawItem] = (await fixture("items")) as unknown[]; + const [rawVote] = (await fixture("votes")) as unknown[]; + const item = adaptDurhamItem(rawItem); + const vote = adaptDurhamVote(rawVote); + + assert.equal(item.consent, true); + assert.equal(item.action, "Approved"); + assert.equal(item.outcome, "Passed"); + assert.equal(item.voteSummary, "5-0"); + assert.equal(item.documents.length, 2); + assert.equal(item.documents[1]?.language, "es"); + assert.equal(vote.voterName, "Commissioner A"); + assert.equal(vote.value, "Aye"); +}); + +void test("uses stable source ids and changes checksum when an amendment changes", async () => { + const [raw] = (await fixture("events")) as Record[]; + const original = adaptDurhamMeeting(raw); + const replaced = adaptDurhamMeeting({ + ...raw, + EventRowVersion: "replacement-v2", + EventLastModifiedUtc: "2026-01-13T00:00:00.000", + }); + + assert.equal(original.externalId, replaced.externalId); + assert.notEqual(original.contentHash, replaced.contentHash); +}); + +void test("parses Durham local time with DST and bounds to the current cycle", () => { + assert.equal( + parseDurhamStart("2026-01-12T00:00:00", "6:00 PM").toISOString(), + "2026-01-12T23:00:00.000Z", + ); + assert.equal( + parseDurhamStart("2026-07-13T00:00:00", "5:00 PM").toISOString(), + "2026-07-13T21:00:00.000Z", + ); + assert.equal( + currentElectionCycleStart(new Date("2026-07-21T00:00:00Z")).toISOString(), + "2025-01-01T00:00:00.000Z", + ); +}); diff --git a/apps/scraper/src/scrapers/durham-bocc.ts b/apps/scraper/src/scrapers/durham-bocc.ts new file mode 100644 index 00000000..0c64762d --- /dev/null +++ b/apps/scraper/src/scrapers/durham-bocc.ts @@ -0,0 +1,539 @@ +import { createHash } from "node:crypto"; +import { z } from "zod/v4"; + +import { and, eq } from "@acme/db"; +import { db } from "@acme/db/client"; +import { + LocalGovernmentAgendaItem, + LocalGovernmentDocument, + LocalGovernmentMeeting, + LocalGovernmentVote, +} from "@acme/db/schema"; + +import type { Scraper } from "../utils/types.js"; +import { getItemLimit } from "../utils/concurrency.js"; +import { setExpectedTotal } from "../utils/db/metrics.js"; +import { fetchWithRetry } from "../utils/fetch.js"; +import { createLogger } from "../utils/log.js"; +import { durhamBoccConfig } from "./durham-bocc.config.js"; + +const API_BASE = "https://webapi.legistar.com/v1/durhamcounty"; +const SITE_BASE = "https://durhamcounty.legistar.com"; +const PROVIDER = "legistar"; +const JURISDICTION = "durham-county-nc"; +const BOCC_BODY_ID = 138; +const SOURCE_VERSION = "durham-legistar-v1"; +const TIMEZONE = "America/New_York"; +const logger = createLogger(durhamBoccConfig.name); + +const attachmentSchema = z + .object({ + MatterAttachmentId: z.number(), + MatterAttachmentName: z.string(), + MatterAttachmentHyperlink: z.string().nullable(), + MatterAttachmentLastModifiedUtc: z.string(), + MatterAttachmentShowOnInternetPage: z.boolean().optional(), + }) + .passthrough(); + +export const durhamEventSchema = z + .object({ + EventId: z.number(), + EventGuid: z.string(), + EventLastModifiedUtc: z.string(), + EventRowVersion: z.string(), + EventBodyId: z.number(), + EventBodyName: z.string(), + EventDate: z.string(), + EventTime: z.string().nullable(), + EventAgendaStatusName: z.string().nullable(), + EventMinutesStatusName: z.string().nullable(), + EventLocation: z.string().nullable(), + EventAgendaFile: z.string().nullable(), + EventMinutesFile: z.string().nullable(), + EventComment: z.string().nullable(), + EventVideoPath: z.string().nullable(), + EventMedia: z.union([z.string(), z.number()]).nullable().optional(), + EventInSiteURL: z.string().nullable(), + }) + .passthrough(); + +export const durhamItemSchema = z + .object({ + EventItemId: z.number(), + EventItemLastModifiedUtc: z.string(), + EventItemRowVersion: z.string(), + EventItemEventId: z.number(), + EventItemAgendaSequence: z.number(), + EventItemAgendaNumber: z.string().nullable(), + EventItemAgendaNote: z.string().nullable(), + EventItemMinutesNote: z.string().nullable(), + EventItemActionName: z.string().nullable(), + EventItemActionText: z.string().nullable(), + EventItemPassedFlagName: z.string().nullable(), + EventItemRollCallFlag: z.number().nullable(), + EventItemTitle: z.string().nullable(), + EventItemTally: z.string().nullable(), + EventItemConsent: z.number(), + EventItemMover: z.string().nullable(), + EventItemSeconder: z.string().nullable(), + EventItemMatterId: z.number().nullable(), + EventItemMatterAttachments: z.array(attachmentSchema).nullable(), + }) + .passthrough(); + +export const durhamVoteSchema = z + .object({ + VoteId: z.number(), + VoteLastModifiedUtc: z.string(), + VotePersonId: z.number(), + VotePersonName: z.string(), + VoteValueName: z.string(), + VoteSort: z.number(), + VoteEventItemId: z.number(), + }) + .passthrough(); + +type DurhamEvent = z.infer; +type DurhamItem = z.infer; +type DurhamVote = z.infer; +interface DurhamDocument { + kind: "agenda" | "minutes" | "attachment"; + title: string; + url: string; + language?: string; +} + +function hash(value: unknown): string { + return createHash("sha256").update(JSON.stringify(value)).digest("hex"); +} + +function datePart(value: string): string { + const match = /^(\d{4}-\d{2}-\d{2})/.exec(value); + if (!match) throw new Error(`Invalid Legistar date: ${value}`); + return match[1]!; +} + +function nthSunday(year: number, monthIndex: number, nth: number): number { + const first = new Date(Date.UTC(year, monthIndex, 1)).getUTCDay(); + return 1 + ((7 - first) % 7) + (nth - 1) * 7; +} + +function easternUtcOffset(date: string): "-04:00" | "-05:00" { + const [year, month, day] = date.split("-").map(Number) as [ + number, + number, + number, + ]; + const dstStart = nthSunday(year, 2, 2); + const dstEnd = nthSunday(year, 10, 1); + const isDst = + (month > 3 && month < 11) || + (month === 3 && day >= dstStart) || + (month === 11 && day < dstEnd); + return isDst ? "-04:00" : "-05:00"; +} + +export function parseDurhamStart( + eventDate: string, + eventTime: string | null, +): Date { + const date = datePart(eventDate); + const match = eventTime?.trim().match(/^(\d{1,2}):(\d{2})\s*([AP]M)$/i); + let hour = 0; + let minute = 0; + if (match) { + hour = Number(match[1]) % 12; + minute = Number(match[2]); + if (match[3]!.toUpperCase() === "PM") hour += 12; + } + return new Date( + `${date}T${String(hour).padStart(2, "0")}:${String(minute).padStart(2, "0")}:00${easternUtcOffset(date)}`, + ); +} + +export function currentElectionCycleStart(now = new Date()): Date { + const year = now.getUTCFullYear(); + const startYear = year % 2 === 0 ? year - 1 : year; + return new Date(Date.UTC(startYear, 0, 1)); +} + +function documentLanguage(title: string): string | undefined { + return /\b(spanish|español|espanol)\b/i.test(title) ? "es" : undefined; +} + +function visibleAttachmentDocuments(item: DurhamItem): DurhamDocument[] { + const kind = /\bminutes?\b/i.test(item.EventItemTitle ?? "") + ? ("minutes" as const) + : ("attachment" as const); + return (item.EventItemMatterAttachments ?? []) + .filter( + (attachment) => + attachment.MatterAttachmentHyperlink && + attachment.MatterAttachmentShowOnInternetPage !== false, + ) + .map((attachment) => ({ + kind, + title: attachment.MatterAttachmentName, + url: attachment.MatterAttachmentHyperlink!, + ...(documentLanguage(attachment.MatterAttachmentName) + ? { language: documentLanguage(attachment.MatterAttachmentName) } + : {}), + })); +} + +export function adaptDurhamMeeting(input: unknown) { + const event = durhamEventSchema.parse(input); + const comment = event.EventComment?.trim() || "Board meeting"; + const markerText = [ + comment, + event.EventAgendaStatusName, + event.EventMinutesStatusName, + event.EventAgendaFile, + event.EventMinutesFile, + ] + .filter(Boolean) + .join(" "); + const isCancelled = /\bcancel(?:led|ed|lation)?\b/i.test(markerText); + const isAmended = /\b(amend(?:ed|ment)?|revis(?:ed|ion)|corrected)\b/i.test( + markerText, + ); + const documents: DurhamDocument[] = []; + if (event.EventAgendaFile) { + documents.push({ + kind: "agenda", + title: `${isAmended ? "Amended " : ""}Agenda`, + url: event.EventAgendaFile, + }); + } + if (event.EventMinutesFile) { + documents.push({ + kind: "minutes", + title: "Minutes", + url: event.EventMinutesFile, + }); + } + const sourceUrl = + event.EventInSiteURL ?? + `${SITE_BASE}/MeetingDetail.aspx?LEGID=${event.EventId}`; + const videoUrl = event.EventVideoPath + ? event.EventVideoPath + : event.EventMedia + ? `${SITE_BASE}/Video.aspx?Mode=Granicus&ID1=${event.EventMedia}&Mode2=Video` + : null; + const status = isCancelled + ? "cancelled" + : (event.EventMinutesStatusName ?? + event.EventAgendaStatusName ?? + "scheduled"); + const mapped = { + source: PROVIDER, + jurisdiction: JURISDICTION, + externalId: String(event.EventId), + governingBody: event.EventBodyName, + title: comment, + meetingType: comment.replace(/^cancel(?:led|ed)\s*-?\s*/i, ""), + startsAt: parseDurhamStart(event.EventDate, event.EventTime), + timezone: TIMEZONE, + location: event.EventLocation, + status, + isCancelled, + isAmended, + canonicalUrl: sourceUrl, + videoUrl, + documents, + sourceVersion: `${SOURCE_VERSION}:${event.EventRowVersion}`, + sourceUpdatedAt: new Date(event.EventLastModifiedUtc), + }; + return { ...mapped, contentHash: hash(mapped) }; +} + +export function adaptDurhamItem(input: unknown) { + const item = durhamItemSchema.parse(input); + const mapped = { + externalId: String(item.EventItemId), + meetingExternalId: String(item.EventItemEventId), + sequence: item.EventItemAgendaSequence, + itemNumber: item.EventItemAgendaNumber, + itemType: item.EventItemActionName ?? "agenda-item", + title: item.EventItemTitle?.trim() || "Untitled agenda item", + description: item.EventItemAgendaNote, + minutesNote: item.EventItemMinutesNote, + consent: item.EventItemConsent === 1, + action: item.EventItemActionName, + motion: item.EventItemActionText, + outcome: item.EventItemPassedFlagName, + voteSummary: item.EventItemTally, + mover: item.EventItemMover, + seconder: item.EventItemSeconder, + documents: visibleAttachmentDocuments(item), + sourceVersion: `${SOURCE_VERSION}:${item.EventItemRowVersion}`, + sourceUpdatedAt: new Date(item.EventItemLastModifiedUtc), + sourceUrl: `${SITE_BASE}/MeetingDetail.aspx?LEGID=${item.EventItemEventId}&GID=174`, + }; + return { ...mapped, contentHash: hash(mapped) }; +} + +export function adaptDurhamVote(input: unknown) { + const vote = durhamVoteSchema.parse(input); + return { + externalId: String(vote.VoteId), + itemExternalId: String(vote.VoteEventItemId), + voterExternalId: String(vote.VotePersonId), + voterName: vote.VotePersonName, + value: vote.VoteValueName, + sort: vote.VoteSort, + sourceUpdatedAt: new Date(vote.VoteLastModifiedUtc), + }; +} + +async function fetchJson(url: URL): Promise { + const response = await fetchWithRetry(url.toString(), { timeoutMs: 30_000 }); + return response.json() as Promise; +} + +async function upsertMeeting(meeting: ReturnType) { + const fetchedAt = new Date(); + const { documents, ...meetingRow } = meeting; + const [stored] = await db + .insert(LocalGovernmentMeeting) + .values({ ...meetingRow, fetchedAt }) + .onConflictDoUpdate({ + target: [ + LocalGovernmentMeeting.source, + LocalGovernmentMeeting.jurisdiction, + LocalGovernmentMeeting.externalId, + ], + set: { ...meetingRow, fetchedAt }, + }) + .returning({ id: LocalGovernmentMeeting.id }); + if (!stored) + throw new Error(`Failed to persist meeting ${meeting.externalId}`); + await upsertDocuments(stored.id, documents); + return stored.id; +} + +async function upsertDocuments(meetingId: string, documents: DurhamDocument[]) { + for (const document of documents) { + await db + .insert(LocalGovernmentDocument) + .values({ + meetingId, + type: document.kind, + title: document.language + ? `${document.title} (${document.language})` + : document.title, + url: document.url, + fetchedAt: new Date(), + }) + .onConflictDoUpdate({ + target: [ + LocalGovernmentDocument.meetingId, + LocalGovernmentDocument.type, + LocalGovernmentDocument.url, + ], + set: { + title: document.language + ? `${document.title} (${document.language})` + : document.title, + isCurrent: true, + fetchedAt: new Date(), + }, + }); + } +} + +async function upsertItem( + meetingId: string, + item: ReturnType, +) { + const { documents } = item; + const itemRow = { + externalId: item.externalId, + sequence: item.sequence, + itemNumber: item.itemNumber, + itemType: item.itemType, + title: item.title, + description: item.description, + minutesNote: item.minutesNote, + consent: item.consent, + action: item.action, + motion: item.motion, + outcome: item.outcome, + voteSummary: item.voteSummary, + mover: item.mover, + seconder: item.seconder, + sourceVersion: item.sourceVersion, + contentHash: item.contentHash, + sourceUpdatedAt: item.sourceUpdatedAt, + sourceUrl: item.sourceUrl, + }; + const [stored] = await db + .insert(LocalGovernmentAgendaItem) + .values({ ...itemRow, meetingId }) + .onConflictDoUpdate({ + target: [ + LocalGovernmentAgendaItem.meetingId, + LocalGovernmentAgendaItem.externalId, + ], + set: itemRow, + }) + .returning({ id: LocalGovernmentAgendaItem.id }); + if (!stored) throw new Error(`Failed to persist item ${item.externalId}`); + await upsertDocuments(meetingId, documents); + return stored.id; +} + +async function upsertVote( + agendaItemId: string, + vote: ReturnType, +) { + const voteRow = { + externalId: vote.externalId, + voterExternalId: vote.voterExternalId, + voterName: vote.voterName, + value: vote.value, + sort: vote.sort, + sourceUpdatedAt: vote.sourceUpdatedAt, + }; + await db + .insert(LocalGovernmentVote) + .values({ ...voteRow, agendaItemId, fetchedAt: new Date() }) + .onConflictDoUpdate({ + target: [LocalGovernmentVote.agendaItemId, LocalGovernmentVote.voterName], + set: { ...voteRow, fetchedAt: new Date() }, + }); +} + +async function scrapeMeeting(rawEvent: unknown): Promise { + const meeting = adaptDurhamMeeting(rawEvent); + const meetingId = await upsertMeeting(meeting); + + const itemsUrl = new URL( + `${API_BASE}/Events/${meeting.externalId}/EventItems`, + ); + itemsUrl.searchParams.set("AgendaNote", "1"); + itemsUrl.searchParams.set("MinutesNote", "1"); + itemsUrl.searchParams.set("Attachments", "1"); + const rawItems = z.array(z.unknown()).parse(await fetchJson(itemsUrl)); + + for (const rawItem of rawItems) { + try { + const parsed = durhamItemSchema.parse(rawItem); + let enrichedItem: unknown = rawItem; + if ( + parsed.EventItemMatterId && + (parsed.EventItemMatterAttachments?.length ?? 0) === 0 + ) { + try { + const attachmentsUrl = new URL( + `${API_BASE}/Matters/${parsed.EventItemMatterId}/Attachments`, + ); + const attachments = z + .array(attachmentSchema) + .parse(await fetchJson(attachmentsUrl)); + enrichedItem = { + ...parsed, + EventItemMatterAttachments: attachments, + }; + } catch (error) { + logger.warn( + `Attachments unavailable for matter ${parsed.EventItemMatterId}`, + error, + ); + } + } + const item = adaptDurhamItem(enrichedItem); + const agendaItemId = await upsertItem(meetingId, item); + if (parsed.EventItemRollCallFlag === 1) { + const votesUrl = new URL( + `${API_BASE}/EventItems/${item.externalId}/Votes`, + ); + const rawVotes = z.array(z.unknown()).parse(await fetchJson(votesUrl)); + for (const rawVote of rawVotes) { + try { + await upsertVote(agendaItemId, adaptDurhamVote(rawVote)); + } catch (error) { + logger.warn( + `Skipping invalid vote for item ${item.externalId}`, + error, + ); + } + } + } + } catch (error) { + logger.warn( + `Skipping invalid item for meeting ${meeting.externalId}`, + error, + ); + } + } + + logger.success(`Synced ${meeting.title} (${meeting.externalId})`); +} + +async function scrape(maxItems = 100): Promise { + const start = currentElectionCycleStart(); + const eventsUrl = new URL(`${API_BASE}/Events`); + eventsUrl.searchParams.set( + "$filter", + `EventDate ge datetime'${start.toISOString().slice(0, 10)}' and EventBodyId eq ${BOCC_BODY_ID}`, + ); + eventsUrl.searchParams.set("$orderby", "EventDate asc"); + eventsUrl.searchParams.set("$top", String(maxItems)); + + logger.info( + `Syncing current election cycle from ${start.toISOString().slice(0, 10)} (structured source; OCR not required)`, + ); + const rawEvents = z + .array(z.unknown()) + .parse(await fetchJson(eventsUrl)) + .slice(0, maxItems); + setExpectedTotal(rawEvents.length); + + const limit = getItemLimit(); + const results = await Promise.allSettled( + rawEvents.map((rawEvent) => + limit(async () => { + try { + await scrapeMeeting(rawEvent); + } catch (error) { + let sourceId = "unknown"; + const parsed = durhamEventSchema.safeParse(rawEvent); + if (parsed.success) sourceId = String(parsed.data.EventId); + logger.error( + `Meeting ${sourceId} failed without aborting run`, + error, + ); + } + }), + ), + ); + const failed = results.filter((result) => result.status === "rejected"); + if (failed.length > 0) logger.warn(`${failed.length} meeting tasks failed`); + logger.success(`Completed ${rawEvents.length} Durham BOCC meetings`); +} + +export const durhamBocc: Scraper = { + ...durhamBoccConfig, + scrape: (options) => + scrape( + (options?.maxItems ?? Number(process.env.DURHAM_BOCC_MAX_ITEMS)) || 100, + ), +}; + +// Exported for targeted cleanup/diagnostics without widening the public API. +export async function hasDurhamMeeting(sourceId: string): Promise { + const rows = await db + .select({ id: LocalGovernmentMeeting.id }) + .from(LocalGovernmentMeeting) + .where( + and( + eq(LocalGovernmentMeeting.source, PROVIDER), + eq(LocalGovernmentMeeting.jurisdiction, JURISDICTION), + eq(LocalGovernmentMeeting.externalId, sourceId), + ), + ) + .limit(1); + return rows.length > 0; +} diff --git a/apps/scraper/src/scrapers/durham-onbase-parser.test.ts b/apps/scraper/src/scrapers/durham-onbase-parser.test.ts new file mode 100644 index 00000000..d55b0c68 --- /dev/null +++ b/apps/scraper/src/scrapers/durham-onbase-parser.test.ts @@ -0,0 +1,73 @@ +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import { describe, it } from "node:test"; + +import { + parseAgendaOutline, + parseItemAttachments, + parseMeetingIndex, +} from "./durham-onbase-parser.js"; + +const fixture = (name: string) => + readFile(new URL(`./__fixtures__/${name}`, import.meta.url), "utf8"); + +describe("Durham OnBase deterministic parsers", () => { + it("parses the embedded meeting index JSON and preserves timezone", async () => { + const meetings = parseMeetingIndex( + await fixture("durham-onbase-index.html"), + ); + assert.equal(meetings.length, 2); + assert.deepEqual( + { + id: meetings[0]?.id, + type: meetings[0]?.meetingType, + iso: meetings[0]?.date.toISOString(), + latestDocumentType: meetings[1]?.latestDocumentType, + }, + { + id: 748, + type: "City Council Meeting Agenda", + iso: "2026-05-18T23:00:00.000Z", + latestDocumentType: 2, + }, + ); + }); + + it("parses sections, items, action text, and vote text", async () => { + const items = parseAgendaOutline( + await fixture("durham-onbase-agenda.html"), + ); + assert.equal(items.length, 2); + assert.deepEqual( + { + externalId: items[0]?.externalId, + section: items[0]?.section, + number: items[0]?.agendaNumber, + title: items[0]?.title, + vote: items[0]?.voteText, + }, + { + externalId: "47768", + section: "Consent Agenda", + number: "1", + title: "Approval of City Council Minutes", + vote: "[Approved by Vote: 7/0]", + }, + ); + assert.match(items[1]?.voteText ?? "", /FAILED by Vote: 0\/7/i); + assert.match(items[1]?.voteText ?? "", /No vote taken/i); + }); + + it("parses stable attachment IDs and absolute official URLs", async () => { + const attachments = parseItemAttachments( + await fixture("durham-onbase-item.html"), + ); + assert.equal(attachments.length, 2); + assert.equal(attachments[0]?.externalId, "272813"); + assert.equal( + new URL(attachments[0]!.url).hostname, + "cityordinances.durhamnc.gov", + ); + assert.equal(attachments[1]?.title, "March 16 City Council Minutes"); + }); +}); diff --git a/apps/scraper/src/scrapers/durham-onbase-parser.ts b/apps/scraper/src/scrapers/durham-onbase-parser.ts new file mode 100644 index 00000000..0764279a --- /dev/null +++ b/apps/scraper/src/scrapers/durham-onbase-parser.ts @@ -0,0 +1,198 @@ +import * as cheerio from "cheerio"; + +export const DURHAM_ONBASE_BASE_URL = + "https://cityordinances.durhamnc.gov/OnBaseAgendaOnline/"; + +export interface OnBaseMeetingIndexItem { + id: number; + name: string; + meetingType: string; + date: Date; + location?: string; + isAgendaAvailable: boolean; + isMinutesAvailable: boolean; + agendaUniqueName: string; + minutesUniqueName: string; + latestDocumentType: 1 | 2 | 3; +} + +export interface ParsedOnBaseAttachment { + externalId: string; + title: string; + url: string; +} + +export interface ParsedOnBaseAgendaItem { + externalId: string; + section?: string; + agendaNumber?: string; + title: string; + actionText?: string; + voteText?: string; + attachments: ParsedOnBaseAttachment[]; + sortOrder: number; +} + +interface RawMeeting { + ID?: unknown; + Name?: unknown; + MeetingTypeName?: unknown; + Time?: unknown; + Location?: unknown; + IsAgendaAvailable?: unknown; + IsMinutesAvailable?: unknown; + AgendaUniqueName?: unknown; + MinutesUniqueName?: unknown; + LatestDocumentType?: unknown; +} + +function cleanText(value: string): string { + return value + .replace(/\u00a0/g, " ") + .replace(/\s+/g, " ") + .trim(); +} + +function extractJsonObject(source: string, start: number): string { + let depth = 0; + let inString = false; + let escaped = false; + + for (let index = start; index < source.length; index++) { + const char = source[index]!; + if (inString) { + if (escaped) escaped = false; + else if (char === "\\") escaped = true; + else if (char === '"') inString = false; + continue; + } + if (char === '"') inString = true; + else if (char === "{") depth++; + else if (char === "}" && --depth === 0) + return source.slice(start, index + 1); + } + + throw new Error("Unterminated OnBase meeting index JSON"); +} + +export function parseMeetingIndex(html: string): OnBaseMeetingIndexItem[] { + const marker = "showSearchResults(new SearchResults("; + const markerIndex = html.indexOf(marker); + if (markerIndex < 0) + throw new Error("OnBase meeting index payload not found"); + const jsonStart = html.indexOf("{", markerIndex + marker.length); + if (jsonStart < 0) throw new Error("OnBase meeting index JSON not found"); + + const payload = JSON.parse(extractJsonObject(html, jsonStart)) as { + Meetings?: RawMeeting[]; + }; + + return (payload.Meetings ?? []).flatMap((raw) => { + if ( + typeof raw.ID !== "number" || + typeof raw.Name !== "string" || + typeof raw.MeetingTypeName !== "string" || + typeof raw.Time !== "string" || + typeof raw.AgendaUniqueName !== "string" || + typeof raw.MinutesUniqueName !== "string" + ) { + return []; + } + const date = new Date(raw.Time); + if (Number.isNaN(date.getTime())) return []; + const latest = + raw.LatestDocumentType === 2 || raw.LatestDocumentType === 3 + ? raw.LatestDocumentType + : 1; + + return [ + { + id: raw.ID, + name: cleanText(raw.Name), + meetingType: cleanText(raw.MeetingTypeName), + date, + location: + typeof raw.Location === "string" && cleanText(raw.Location) + ? cleanText(raw.Location) + : undefined, + isAgendaAvailable: raw.IsAgendaAvailable === true, + isMinutesAvailable: raw.IsMinutesAvailable === true, + agendaUniqueName: raw.AgendaUniqueName, + minutesUniqueName: raw.MinutesUniqueName, + latestDocumentType: latest, + }, + ]; + }); +} + +function extractVoteText(actionText: string): string | undefined { + const matches = actionText.match( + /\[(?:approved|failed)[^\]]*\]|\[motion referred[^\]]*\]|no vote (?:was )?taken\.?|ayes:\s*[^.]*\.?|nays:\s*[^.]*\.?/gi, + ); + return matches ? cleanText(matches.join(" ")) : undefined; +} + +export function parseAgendaOutline(html: string): ParsedOnBaseAgendaItem[] { + const $ = cheerio.load(html); + const items: ParsedOnBaseAgendaItem[] = []; + let currentSection: string | undefined; + + $("a[href*='loadAgendaItem']").each((_, element) => { + const anchor = $(element); + const href = anchor.attr("href") ?? ""; + const match = /loadAgendaItem\((\d+),(true|false)\)/.exec(href); + if (!match) return; + const title = cleanText(anchor.text()); + if (!title) return; + if (match[2] === "true") { + currentSection = title; + return; + } + + const externalId = match[1]!; + const agendaMatch = /^(\d+[A-Za-z]?)\.\s*(.+)$/.exec(title); + const displayTitle = agendaMatch?.[2] ?? title; + const tableText = cleanText(anchor.closest("table").text()); + const actionText = cleanText( + tableText.startsWith(title) ? tableText.slice(title.length) : tableText, + ); + + items.push({ + externalId, + section: currentSection, + agendaNumber: agendaMatch?.[1], + title: displayTitle, + actionText: actionText || undefined, + voteText: actionText ? extractVoteText(actionText) : undefined, + attachments: [], + sortOrder: items.length, + }); + }); + + return items; +} + +export function parseItemAttachments( + html: string, + baseUrl = DURHAM_ONBASE_BASE_URL, +): ParsedOnBaseAttachment[] { + const $ = cheerio.load(html); + return $("a[href*='/Documents/DownloadFile/']") + .toArray() + .flatMap((element) => { + const anchor = $(element); + const href = anchor.attr("href"); + if (!href) return []; + const url = new URL(href, baseUrl); + const externalId = + url.searchParams.get("publishId") ?? + (anchor.attr("id")?.match(/(\d+)$/)?.[1] || url.toString()); + return [ + { + externalId, + title: cleanText(anchor.text()), + url: url.toString(), + }, + ]; + }); +} diff --git a/apps/scraper/src/scrapers/durham-onbase.config.ts b/apps/scraper/src/scrapers/durham-onbase.config.ts new file mode 100644 index 00000000..9297fbf7 --- /dev/null +++ b/apps/scraper/src/scrapers/durham-onbase.config.ts @@ -0,0 +1,11 @@ +import type { ScraperEnvContract } from "@acme/env"; + +export const durhamOnBaseConfig = { + id: "durham-onbase", + name: "Durham City Council OnBase", + source: "City of Durham OnBase Agenda Online", + environment: { + required: ["POSTGRES_URL"], + optional: ["DURHAM_ONBASE_MAX_ITEMS", "DURHAM_ONBASE_CACHE_TTL_HOURS"], + }, +} as const satisfies ScraperEnvContract; diff --git a/apps/scraper/src/scrapers/durham-onbase.ts b/apps/scraper/src/scrapers/durham-onbase.ts new file mode 100644 index 00000000..385963f1 --- /dev/null +++ b/apps/scraper/src/scrapers/durham-onbase.ts @@ -0,0 +1,302 @@ +import { createHash } from "node:crypto"; +import { and, eq, notInArray } from "drizzle-orm"; + +import { db } from "@acme/db/client"; +import { + LocalGovernmentAgendaItem, + LocalGovernmentDocument, + LocalGovernmentMeeting, +} from "@acme/db/schema"; + +import type { Scraper } from "../utils/types.js"; +import { getItemLimit } from "../utils/concurrency.js"; +import { setExpectedTotal } from "../utils/db/metrics.js"; +import { fetchWithRetry } from "../utils/fetch.js"; +import { createLogger } from "../utils/log.js"; +import { + DURHAM_ONBASE_BASE_URL, + parseAgendaOutline, + parseItemAttachments, + parseMeetingIndex, +} from "./durham-onbase-parser.js"; +import { durhamOnBaseConfig } from "./durham-onbase.config.js"; + +const PROVIDER = "onbase"; +const JURISDICTION = "durham-nc"; +const GOVERNING_BODY = "Durham City Council"; +const SOURCE_VERSION = "onbase-agenda-online-v1"; +const MIN_REQUEST_INTERVAL_MS = 250; +const logger = createLogger(durhamOnBaseConfig.name); +let nextRequestAt = 0; +let requestGate = Promise.resolve(); + +async function fetchOnBase(path: string): Promise { + const turn = requestGate.then(async () => { + const delay = Math.max(0, nextRequestAt - Date.now()); + if (delay) await new Promise((resolve) => setTimeout(resolve, delay)); + nextRequestAt = Date.now() + MIN_REQUEST_INTERVAL_MS; + }); + requestGate = turn.catch(() => undefined); + await turn; + const response = await fetchWithRetry( + new URL(path, DURHAM_ONBASE_BASE_URL).toString(), + { + timeoutMs: 30_000, + headers: { + "User-Agent": + "Billion civic-data scraper (contact: support@billion.app)", + }, + }, + ); + return response.text(); +} + +function durhamCalendarYear(date: Date): number { + return Number( + new Intl.DateTimeFormat("en-US", { + year: "numeric", + timeZone: "America/New_York", + }).format(date), + ); +} + +function documentTypeName(documentType: 1 | 2 | 3): string { + if (documentType === 2) return "minutes"; + if (documentType === 3) return "summary"; + return "agenda"; +} + +function pdfUrl( + uniqueName: string, + documentType: number, + meetingId: number, +): string { + const path = `Documents/DownloadFile/${encodeURIComponent(uniqueName)}.pdf`; + const url = new URL(path, DURHAM_ONBASE_BASE_URL); + url.searchParams.set("documentType", String(documentType)); + url.searchParams.set("meetingId", String(meetingId)); + return url.toString(); +} + +function contentHash(value: unknown): string { + return createHash("sha256").update(JSON.stringify(value)).digest("hex"); +} + +async function scrape(maxItems = 100): Promise { + const cycleYear = new Date().getFullYear(); + const ttlHours = Number(process.env.DURHAM_ONBASE_CACHE_TTL_HOURS ?? 24); + const cacheCutoff = new Date(Date.now() - ttlHours * 60 * 60 * 1000); + logger.info(`Fetching current ${cycleYear} council cycle`); + + const indexHtml = await fetchOnBase(""); + const meetings = parseMeetingIndex(indexHtml) + .filter((meeting) => durhamCalendarYear(meeting.date) === cycleYear) + .slice(0, maxItems); + setExpectedTotal(meetings.length); + + const limit = getItemLimit(); + const results = await Promise.allSettled( + meetings.map((meeting) => + limit(async () => { + const externalId = String(meeting.id); + const [cached] = await db + .select({ fetchedAt: LocalGovernmentMeeting.fetchedAt }) + .from(LocalGovernmentMeeting) + .where( + and( + eq(LocalGovernmentMeeting.source, PROVIDER), + eq(LocalGovernmentMeeting.jurisdiction, JURISDICTION), + eq(LocalGovernmentMeeting.externalId, externalId), + ), + ) + .limit(1); + if (cached && cached.fetchedAt >= cacheCutoff) { + logger.info(`Cached: ${meeting.name}`); + return; + } + + const documentType = meeting.latestDocumentType; + const type = documentTypeName(documentType); + const outlineHtml = await fetchOnBase( + `Documents/ViewAgenda?meetingId=${meeting.id}&type=${type}&doctype=${documentType}`, + ); + const items = parseAgendaOutline(outlineHtml); + + for (const item of items) { + const detailHtml = await fetchOnBase( + `Meetings/ViewMeetingAgendaItem?meetingId=${meeting.id}&itemId=${item.externalId}&isSection=false&type=${type}`, + ); + item.attachments = parseItemAttachments(detailHtml); + } + + const sourceUrl = new URL( + `Meetings/ViewMeeting?doctype=${documentType}&id=${meeting.id}`, + DURHAM_ONBASE_BASE_URL, + ).toString(); + const [storedMeeting] = await db + .insert(LocalGovernmentMeeting) + .values({ + source: PROVIDER, + sourceVersion: SOURCE_VERSION, + jurisdiction: JURISDICTION, + governingBody: GOVERNING_BODY, + externalId, + title: meeting.name, + meetingType: meeting.meetingType, + status: documentType === 2 ? "minutes-published" : "published", + startsAt: meeting.date, + location: meeting.location, + canonicalUrl: sourceUrl, + contentHash: contentHash({ meeting, items }), + fetchedAt: new Date(), + }) + .onConflictDoUpdate({ + target: [ + LocalGovernmentMeeting.source, + LocalGovernmentMeeting.jurisdiction, + LocalGovernmentMeeting.externalId, + ], + set: { + sourceVersion: SOURCE_VERSION, + governingBody: GOVERNING_BODY, + title: meeting.name, + meetingType: meeting.meetingType, + status: documentType === 2 ? "minutes-published" : "published", + startsAt: meeting.date, + location: meeting.location, + canonicalUrl: sourceUrl, + contentHash: contentHash({ meeting, items }), + fetchedAt: new Date(), + }, + }) + .returning({ id: LocalGovernmentMeeting.id }); + if (!storedMeeting) + throw new Error(`Failed to persist meeting ${externalId}`); + + const documents = [ + ...(meeting.isAgendaAvailable + ? [ + { + type: "agenda", + title: `${meeting.name} agenda`, + url: pdfUrl(meeting.agendaUniqueName, 1, meeting.id), + }, + ] + : []), + ...(meeting.isMinutesAvailable + ? [ + { + type: "minutes", + title: `${meeting.name} minutes`, + url: pdfUrl(meeting.minutesUniqueName, 2, meeting.id), + }, + ] + : []), + ...items.flatMap((item) => + item.attachments.map((attachment) => ({ + type: "attachment", + title: attachment.title, + url: attachment.url, + })), + ), + ]; + for (const document of documents) { + await db + .insert(LocalGovernmentDocument) + .values({ + meetingId: storedMeeting.id, + ...document, + mediaType: document.url.toLowerCase().endsWith(".pdf") + ? "application/pdf" + : undefined, + fetchedAt: new Date(), + }) + .onConflictDoUpdate({ + target: [ + LocalGovernmentDocument.meetingId, + LocalGovernmentDocument.type, + LocalGovernmentDocument.url, + ], + set: { + title: document.title, + isCurrent: true, + fetchedAt: new Date(), + }, + }); + } + + for (const item of items) { + const itemSourceUrl = new URL( + `Meetings/ViewMeetingAgendaItem?meetingId=${meeting.id}&itemId=${item.externalId}&isSection=false&type=${type}`, + DURHAM_ONBASE_BASE_URL, + ).toString(); + await db + .insert(LocalGovernmentAgendaItem) + .values({ + meetingId: storedMeeting.id, + externalId: item.externalId, + sequence: item.sortOrder, + itemNumber: item.agendaNumber, + section: item.section, + itemType: "agenda-item", + title: item.title, + motion: item.actionText, + voteSummary: item.voteText, + sourceUrl: itemSourceUrl, + }) + .onConflictDoUpdate({ + target: [ + LocalGovernmentAgendaItem.meetingId, + LocalGovernmentAgendaItem.externalId, + ], + set: { + sequence: item.sortOrder, + itemNumber: item.agendaNumber, + section: item.section, + title: item.title, + motion: item.actionText, + voteSummary: item.voteText, + sourceUrl: itemSourceUrl, + }, + }); + } + + if (items.length) { + await db.delete(LocalGovernmentAgendaItem).where( + and( + eq(LocalGovernmentAgendaItem.meetingId, storedMeeting.id), + notInArray( + LocalGovernmentAgendaItem.externalId, + items.map((item) => item.externalId), + ), + ), + ); + } else { + await db + .delete(LocalGovernmentAgendaItem) + .where(eq(LocalGovernmentAgendaItem.meetingId, storedMeeting.id)); + } + logger.success(`Scraped ${meeting.name} (${items.length} items)`); + }), + ), + ); + + const failures = results.filter( + (result): result is PromiseRejectedResult => result.status === "rejected", + ); + if (failures.length) { + throw new AggregateError( + failures.map((failure) => failure.reason), + `${failures.length} Durham meeting(s) failed`, + ); + } + logger.success(`Completed ${meetings.length} current-cycle meetings`); +} + +export const durhamOnBase: Scraper = { + ...durhamOnBaseConfig, + scrape: (options) => + scrape( + options?.maxItems ?? Number(process.env.DURHAM_ONBASE_MAX_ITEMS ?? 100), + ), +}; diff --git a/apps/scraper/src/scrapers/fixtures/civicengage/agenda.txt b/apps/scraper/src/scrapers/fixtures/civicengage/agenda.txt new file mode 100644 index 00000000..b83450c8 --- /dev/null +++ b/apps/scraper/src/scrapers/fixtures/civicengage/agenda.txt @@ -0,0 +1,10 @@ +CITY COUNCIL AGENDA +Consent Agenda +D.1 Approval Of Minutes From The Regular Scheduled City Council Meeting On December 18, 2025. +E.1 Second Reading And Approval Of An Ordinance Calling And Ordering A Special Called Election To Be Held On May 2, 2026. +F.1 A Resolution To Adopt The Cedar Park Transportation Criteria Manual (TCM). +Public Hearings +G.1 First Reading And Public Hearing Of An Ordinance For A Future Land Use Plan Amendment. +Regular Agenda (Non-Consent) +H.1 Consideration Of A Resolution Authorizing A Grant Agreement With The Lower Colorado River Authority In The Amount Of $100,000. +H.2 Consider Action, If Any, On Items Discussed In Executive Session. diff --git a/apps/scraper/src/scrapers/fixtures/civicengage/meetings.html b/apps/scraper/src/scrapers/fixtures/civicengage/meetings.html new file mode 100644 index 00000000..c907d64f --- /dev/null +++ b/apps/scraper/src/scrapers/fixtures/civicengage/meetings.html @@ -0,0 +1,44 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
City Council Mtg.2/12/20266:00 PMCEDAR PARK CITY HALL COUNCIL CHAMBERSPDF Agenda for February 12, 2026 City Council Mtg.PDF Packet for February 12, 2026 City Council Mtg.PDF Minutes for February 12, 2026 City Council Mtg.
City Council - Special Called2/25/20265:00 PMCITY HALLAgenda
City Council Mtg. - CANCELLED3/12/20266:00 PMCITY HALLAgenda
City Council Mtg. - Amended3/13/20257:00 PMCITY HALLAgenda
City Council Mtg.3/13/20257:00 PMCITY HALLPrevious Agenda
Planning and Zoning Commission3/1/20266:00 PMCITY HALL
diff --git a/apps/scraper/src/scrapers/fixtures/civicengage/minutes.txt b/apps/scraper/src/scrapers/fixtures/civicengage/minutes.txt new file mode 100644 index 00000000..32f53f9b --- /dev/null +++ b/apps/scraper/src/scrapers/fixtures/civicengage/minutes.txt @@ -0,0 +1,24 @@ +CITY COUNCIL MINUTES +Consent Agenda +D.1 Approval Of Minutes From The Regular Scheduled City Council Meeting On December 18, 2025. +Approved under the Consent Agenda. +E.1 Second Reading And Approval Of An Ordinance Calling And Ordering A Special Called Election To Be Held On May 2, 2026. +Approved under the Consent Agenda. +F.1 A Resolution To Adopt The Cedar Park Transportation Criteria Manual (TCM). +Motion to approve Agenda Item F.1 as presented. +Movant: Councilmember Kirkland +Second: Councilmember Frezza +Vote: 6-0 with Mayor Pro Tem Boyce absent from meeting +Public Hearings +G.1 First Reading And Public Hearing Of An Ordinance For A Future Land Use Plan Amendment. +Mayor opened the Public Hearing. No Public Comment. Mayor closed the Public Hearing. +Regular Agenda (Non-Consent) +H.1 Consideration Of A Resolution Authorizing A Grant Agreement With The Lower Colorado River Authority In The Amount Of $100,000. +Motion to approve Agenda Item H.1 as presented. +Movant: Councilmember Kirkland +Second: Councilmember Duffy +Vote: 5-1. +Voting Aye: Kirkland, Duffy, Harris, Frezza, and Penniman-Morin +Voting Nay: Darby +H.2 Consider Action, If Any, On Items Discussed In Executive Session. +No action taken. diff --git a/apps/scraper/src/scrapers/fixtures/durham-bocc/events.json b/apps/scraper/src/scrapers/fixtures/durham-bocc/events.json new file mode 100644 index 00000000..fe275d86 --- /dev/null +++ b/apps/scraper/src/scrapers/fixtures/durham-bocc/events.json @@ -0,0 +1,78 @@ +[ + { + "EventId": 1439, + "EventGuid": "4AF37432-715B-4E88-8C23-C24F0ABF3D98", + "EventLastModifiedUtc": "2026-01-12T20:19:16.093", + "EventRowVersion": "AAAAAACFTNc=", + "EventBodyId": 138, + "EventBodyName": "Board of County Commissioners", + "EventDate": "2026-01-12T00:00:00", + "EventTime": "6:00 PM", + "EventAgendaStatusName": "Final", + "EventMinutesStatusName": "Draft", + "EventLocation": "Commissioners' Chambers", + "EventAgendaFile": "https://durhamcounty.legistar1.com/durhamcounty/meetings/2026/1/1439_Agenda.pdf", + "EventMinutesFile": null, + "EventComment": "Regular Session", + "EventVideoPath": null, + "EventMedia": "1539", + "EventInSiteURL": "https://durhamcounty.legistar.com/MeetingDetail.aspx?LEGID=1439" + }, + { + "EventId": 1441, + "EventGuid": "AA819B77-C3F4-49A5-8BF0-29BCDDD09B25", + "EventLastModifiedUtc": "2026-02-02T12:55:38.687", + "EventRowVersion": "AAAAAACFhUA=", + "EventBodyId": 138, + "EventBodyName": "Board of County Commissioners", + "EventDate": "2026-02-02T00:00:00", + "EventTime": "9:00 AM", + "EventAgendaStatusName": "Final", + "EventMinutesStatusName": "Draft", + "EventLocation": "Commissioners' Chambers", + "EventAgendaFile": "https://durhamcounty.legistar1.com/durhamcounty/meetings/2026/2/1441_Agenda.pdf", + "EventMinutesFile": null, + "EventComment": "Work Session", + "EventVideoPath": null, + "EventMedia": null, + "EventInSiteURL": "https://durhamcounty.legistar.com/MeetingDetail.aspx?LEGID=1441" + }, + { + "EventId": 1440, + "EventGuid": "418F96F6-84B8-40B6-9BD8-F409555EDFD7", + "EventLastModifiedUtc": "2026-01-26T01:33:13.067", + "EventRowVersion": "AAAAAACFIbU=", + "EventBodyId": 138, + "EventBodyName": "Board of County Commissioners", + "EventDate": "2026-01-26T00:00:00", + "EventTime": "7:00 PM", + "EventAgendaStatusName": "Final", + "EventMinutesStatusName": "Draft", + "EventLocation": "Commissioners' Chambers", + "EventAgendaFile": "https://durhamcounty.legistar1.com/durhamcounty/meetings/2026/1/1440_Agenda.pdf", + "EventMinutesFile": null, + "EventComment": "Cancelled - Regular Session", + "EventVideoPath": null, + "EventMedia": null, + "EventInSiteURL": "https://durhamcounty.legistar.com/MeetingDetail.aspx?LEGID=1440" + }, + { + "EventId": 1499, + "EventGuid": "EDC0142F-3923-4A79-AAA7-9319E811C112", + "EventLastModifiedUtc": "2026-05-04T15:00:00.000", + "EventRowVersion": "AMENDED-V2", + "EventBodyId": 138, + "EventBodyName": "Board of County Commissioners", + "EventDate": "2026-05-04T00:00:00", + "EventTime": "9:00 AM", + "EventAgendaStatusName": "Final", + "EventMinutesStatusName": "Draft", + "EventLocation": "Commissioners' Chambers", + "EventAgendaFile": "https://durhamcounty.legistar1.com/durhamcounty/meetings/2026/5/1499_Amended_Agenda.pdf", + "EventMinutesFile": null, + "EventComment": "Amended Work Session", + "EventVideoPath": null, + "EventMedia": "1599", + "EventInSiteURL": "https://durhamcounty.legistar.com/MeetingDetail.aspx?LEGID=1499" + } +] diff --git a/apps/scraper/src/scrapers/fixtures/durham-bocc/items.json b/apps/scraper/src/scrapers/fixtures/durham-bocc/items.json new file mode 100644 index 00000000..cfc2a009 --- /dev/null +++ b/apps/scraper/src/scrapers/fixtures/durham-bocc/items.json @@ -0,0 +1,38 @@ +[ + { + "EventItemId": 30280, + "EventItemLastModifiedUtc": "2026-04-14T13:52:55.327", + "EventItemRowVersion": "ITEM-V1", + "EventItemEventId": 1439, + "EventItemAgendaSequence": 12, + "EventItemAgendaNumber": "6.", + "EventItemAgendaNote": "Approve the consent agenda.", + "EventItemMinutesNote": "The motion carried unanimously.", + "EventItemActionName": "Approved", + "EventItemActionText": "Adopt the consent agenda", + "EventItemPassedFlagName": "Passed", + "EventItemRollCallFlag": 1, + "EventItemTitle": "Consent Agenda", + "EventItemTally": "5-0", + "EventItemConsent": 1, + "EventItemMover": "Commissioner A", + "EventItemSeconder": "Commissioner B", + "EventItemMatterId": 9181, + "EventItemMatterAttachments": [ + { + "MatterAttachmentId": 22223, + "MatterAttachmentName": "Budget memorandum", + "MatterAttachmentHyperlink": "https://durhamcounty.legistar1.com/durhamcounty/attachments/budget.pdf", + "MatterAttachmentLastModifiedUtc": "2026-04-09T14:39:50.21", + "MatterAttachmentShowOnInternetPage": true + }, + { + "MatterAttachmentId": 22224, + "MatterAttachmentName": "Presupuesto - Español", + "MatterAttachmentHyperlink": "https://durhamcounty.legistar1.com/durhamcounty/attachments/presupuesto.pdf", + "MatterAttachmentLastModifiedUtc": "2026-04-09T14:40:50.21", + "MatterAttachmentShowOnInternetPage": true + } + ] + } +] diff --git a/apps/scraper/src/scrapers/fixtures/durham-bocc/votes.json b/apps/scraper/src/scrapers/fixtures/durham-bocc/votes.json new file mode 100644 index 00000000..55e7c2c7 --- /dev/null +++ b/apps/scraper/src/scrapers/fixtures/durham-bocc/votes.json @@ -0,0 +1,11 @@ +[ + { + "VoteId": 9001, + "VoteLastModifiedUtc": "2026-04-14T14:00:00.000", + "VotePersonId": 101, + "VotePersonName": "Commissioner A", + "VoteValueName": "Aye", + "VoteSort": 1, + "VoteEventItemId": 30280 + } +] diff --git a/apps/scraper/src/scrapers/fixtures/texas-hb9-history.xml b/apps/scraper/src/scrapers/fixtures/texas-hb9-history.xml new file mode 100644 index 00000000..b92063fa --- /dev/null +++ b/apps/scraper/src/scrapers/fixtures/texas-hb9-history.xml @@ -0,0 +1,78 @@ + + + Relating to an exemption from ad valorem taxation. + Meyer | Bonnen + Bernal + Bettencourt + + + Taxation--Property-Exemptions (I0793) + Business & Commerce--General (I0050) + + + + + + + + + + + Introduced + https://capitol.texas.gov/tlodocs/89R/billtext/html/HB00009I.htm + https://capitol.texas.gov/tlodocs/89R/billtext/pdf/HB00009I.pdf + ftp://ftp.legis.state.tx.us/bills/89R/billtext/HTML/house_bills/HB00001_HB00099/HB00009I.HTM + ftp://ftp.legis.state.tx.us/bills/89R/billtext/PDF/house_bills/HB00001_HB00099/HB00009I.PDF + + + Enrolled + https://capitol.texas.gov/tlodocs/89R/billtext/html/HB00009F.htm + https://capitol.texas.gov/tlodocs/89R/billtext/pdf/HB00009F.pdf + ftp://ftp.legis.state.tx.us/bills/89R/billtext/HTML/house_bills/HB00001_HB00099/HB00009F.HTM + ftp://ftp.legis.state.tx.us/bills/89R/billtext/PDF/house_bills/HB00001_HB00099/HB00009F.PDF + + + + + + + House Committee Report + https://capitol.texas.gov/tlodocs/89R/analysis/html/HB00009H.htm + https://capitol.texas.gov/tlodocs/89R/analysis/pdf/HB00009H.pdf + ftp://ftp.legis.state.tx.us/bills/89R/analysis/HTML/house_bills/HB00001_HB00099/HB00009H.HTM + ftp://ftp.legis.state.tx.us/bills/89R/analysis/PDF/house_bills/HB00001_HB00099/HB00009H.PDF + + + + + + + Introduced + https://capitol.texas.gov/tlodocs/89R/fiscalnotes/html/HB00009I.htm + https://capitol.texas.gov/tlodocs/89R/fiscalnotes/pdf/HB00009I.pdf + ftp://ftp.legis.state.tx.us/bills/89R/fiscalNotes/HTML/house_bills/HB00001_HB00099/HB00009I.HTM + ftp://ftp.legis.state.tx.us/bills/89R/fiscalNotes/PDF/house_bills/HB00001_HB00099/HB00009I.PDF + + + + + + + + 06/12/2025 + E100 + Effective immediately + + + 11/12/2024 + H001 + Filed + + + 05/19/2025 + H630 + Record vote + RV#2999 + + + diff --git a/apps/scraper/src/scrapers/ncsbe-parsers.test.ts b/apps/scraper/src/scrapers/ncsbe-parsers.test.ts new file mode 100644 index 00000000..c9abd4d0 --- /dev/null +++ b/apps/scraper/src/scrapers/ncsbe-parsers.test.ts @@ -0,0 +1,58 @@ +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import test from "node:test"; + +import { + parseCandidateCsv, + parseReferendumLines, + parseResultsTsv, + restrictToCycle, +} from "./ncsbe-parsers.js"; + +const fixture = (name: string) => + readFile(new URL(`../fixtures/ncsbe/${name}`, import.meta.url), "utf8"); + +test("parses current-cycle candidate CSV without private contact fields", async () => { + const rows = parseCandidateCsv(await fixture("candidates-2026.csv")); + assert.equal(rows.length, 2); + assert.deepEqual(rows[0], { + electionDate: "2026-03-03", + county: "DURHAM", + contest: "US HOUSE OF REPRESENTATIVES DISTRICT 04", + name: "Nida Allam", + party: "DEM", + voteFor: 1, + termYears: 2, + hasPrimary: true, + isPartisan: true, + }); + assert.equal("email" in rows[0]!, false); + assert.deepEqual( + rows.map((row) => row.county), + ["DURHAM", "WAKE"], + ); +}); + +test("parses NCSBE result layouts from 2026 and 2024 fixtures", async () => { + const current = parseResultsTsv(await fixture("results-2026.tsv")); + const legacy = parseResultsTsv(await fixture("results-2024.tsv")); + assert.equal(current.length, 2); + assert.equal(current[0]?.totalVotes, 371); + assert.equal(current[0]?.earlyVotingVotes, 0); + assert.equal(legacy.length, 2); + assert.equal(legacy[0]?.choice, "Kamala D. Harris"); + assert.deepEqual(restrictToCycle([...current, ...legacy], 2026), current); +}); + +test("parses current-cycle referendum PDF text across counties", async () => { + const rows = parseReferendumLines( + (await fixture("referendums-2026.txt")).split(/\r?\n/), + ); + assert.equal(rows.length, 4); + assert.deepEqual( + [...new Set(rows.map((row) => row.county))], + ["GATES", "GRANVILLE"], + ); + assert.equal(rows[0]?.electionDate, "2026-03-03"); + assert.equal(rows[0]?.choice, "For"); +}); diff --git a/apps/scraper/src/scrapers/ncsbe-parsers.ts b/apps/scraper/src/scrapers/ncsbe-parsers.ts new file mode 100644 index 00000000..bcfa3d76 --- /dev/null +++ b/apps/scraper/src/scrapers/ncsbe-parsers.ts @@ -0,0 +1,328 @@ +import { inflateRawSync } from "node:zlib"; + +export const NCSBE_STRUCTURE_VERSION = "ncsbe-public-election-v1"; + +export interface NcsbeCandidateRecord { + electionDate: string; + county: string; + contest: string; + name: string; + party: string | null; + voteFor: number | null; + termYears: number | null; + hasPrimary: boolean | null; + isPartisan: boolean | null; +} + +export interface NcsbeReferendumRecord { + electionDate: string; + county: string; + contest: string; + choice: string; + description: string | null; +} + +export interface NcsbeResultRecord { + electionDate: string; + county: string; + precinct: string; + contestId: string | null; + contestType: string | null; + contest: string; + choice: string; + party: string | null; + voteFor: number | null; + electionDayVotes: number; + earlyVotingVotes: number; + absenteeMailVotes: number; + provisionalVotes: number; + totalVotes: number; + realPrecinct: boolean | null; +} + +function clean(value: string | undefined): string { + return (value ?? "").replace(/^\uFEFF/, "").trim(); +} + +function nullable(value: string | undefined): string | null { + const result = clean(value); + return result ? result : null; +} + +function integer(value: string | undefined): number | null { + const normalized = clean(value).replace(/,/g, ""); + if (!normalized) return null; + const result = Number(normalized); + return Number.isSafeInteger(result) ? result : null; +} + +function boolean(value: string | undefined): boolean | null { + const normalized = clean(value).toLowerCase(); + if (["true", "t", "yes", "y", "1"].includes(normalized)) return true; + if (["false", "f", "no", "n", "0"].includes(normalized)) return false; + return null; +} + +export function normalizeElectionDate(value: string): string | null { + const normalized = clean(value); + const slash = /^(\d{1,2})\/(\d{1,2})\/(\d{4})$/.exec(normalized); + if (slash) { + return `${slash[3]}-${slash[1]!.padStart(2, "0")}-${slash[2]!.padStart(2, "0")}`; + } + return /^\d{4}-\d{2}-\d{2}$/.test(normalized) ? normalized : null; +} + +/** Small RFC 4180 parser; NCSBE files use commas, quoted fields and CRLF. */ +export function parseDelimited(text: string, delimiter: string): string[][] { + const rows: string[][] = []; + let row: string[] = []; + let field = ""; + let quoted = false; + + for (let i = 0; i < text.length; i++) { + const char = text[i]!; + if (quoted) { + if (char === '"' && text[i + 1] === '"') { + field += '"'; + i++; + } else if (char === '"') { + quoted = false; + } else { + field += char; + } + } else if (char === '"') { + quoted = true; + } else if (char === delimiter) { + row.push(field); + field = ""; + } else if (char === "\n") { + row.push(field.replace(/\r$/, "")); + if (row.some((cell) => cell.length > 0)) rows.push(row); + row = []; + field = ""; + } else { + field += char; + } + } + if (field.length > 0 || row.length > 0) { + row.push(field.replace(/\r$/, "")); + if (row.some((cell) => cell.length > 0)) rows.push(row); + } + return rows; +} + +function records(text: string, delimiter: string): Record[] { + const [rawHeaders, ...rows] = parseDelimited(text, delimiter); + if (!rawHeaders) return []; + const headers = rawHeaders.map((header) => clean(header).toLowerCase()); + return rows.map((row) => + Object.fromEntries( + headers.map((header, index) => [header, row[index] ?? ""]), + ), + ); +} + +function first(row: Record, ...keys: string[]): string { + for (const key of keys) { + if (row[key] !== undefined) return row[key]; + } + return ""; +} + +export function parseCandidateCsv(text: string): NcsbeCandidateRecord[] { + return records(text, ",").flatMap((row) => { + const electionDate = normalizeElectionDate( + first(row, "election_dt", "election_date"), + ); + const county = clean(first(row, "county_name", "county")); + const contest = clean(first(row, "contest_name", "contest")); + const name = clean( + first(row, "name_on_ballot", "candidate_name", "choice"), + ); + if (!electionDate || !county || !contest || !name) return []; + return [ + { + electionDate, + county, + contest, + name, + party: nullable( + first(row, "party_candidate", "candidate_party", "party"), + ), + voteFor: integer(first(row, "vote_for")), + termYears: integer(first(row, "term", "term_years")), + hasPrimary: boolean(first(row, "has_primary")), + isPartisan: boolean(first(row, "is_partisan")), + }, + ]; + }); +} + +export function parseResultsTsv(text: string): NcsbeResultRecord[] { + return records(text, "\t").flatMap((row) => { + const electionDate = normalizeElectionDate( + first(row, "election date", "election_date"), + ); + const county = clean(first(row, "county")); + const precinct = clean(first(row, "precinct")); + const contest = clean(first(row, "contest name", "contest_name")); + const choice = clean(first(row, "choice", "candidate")); + const totals = [ + integer(first(row, "election day", "election_day")), + integer(first(row, "early voting", "early_voting")), + integer(first(row, "absentee by mail", "absentee_by_mail")), + integer(first(row, "provisional")), + integer(first(row, "total votes", "total_votes")), + ]; + if ( + !electionDate || + !county || + !precinct || + !contest || + !choice || + totals.some((n) => n === null) + ) { + return []; + } + return [ + { + electionDate, + county, + precinct, + contestId: nullable(first(row, "contest group id", "contest_group_id")), + contestType: nullable(first(row, "contest type", "contest_type")), + contest, + choice, + party: nullable(first(row, "choice party", "choice_party", "party")), + voteFor: integer(first(row, "vote for", "vote_for")), + electionDayVotes: totals[0]!, + earlyVotingVotes: totals[1]!, + absenteeMailVotes: totals[2]!, + provisionalVotes: totals[3]!, + totalVotes: totals[4]!, + realPrecinct: boolean(first(row, "real precinct", "real_precinct")), + }, + ]; + }); +} + +const NC_COUNTIES = new Set( + `ALAMANCE ALEXANDER ALLEGHANY ANSON ASHE AVERY BEAUFORT BERTIE BLADEN BRUNSWICK BUNCOMBE BURKE CABARRUS CALDWELL CAMDEN CARTERET CASWELL CATAWBA CHATHAM CHEROKEE CHOWAN CLAY CLEVELAND COLUMBUS CRAVEN CUMBERLAND CURRITUCK DARE DAVIDSON DAVIE DUPLIN DURHAM EDGECOMBE FORSYTH FRANKLIN GASTON GATES GRAHAM GRANVILLE GREENE GUILFORD HALIFAX HARNETT HAYWOOD HENDERSON HERTFORD HOKE HYDE IREDELL JACKSON JOHNSTON JONES LEE LENOIR LINCOLN MACON MADISON MARTIN MCDOWELL MECKLENBURG MITCHELL MONTGOMERY MOORE NASH NEW HANOVER NORTHAMPTON ONSLOW ORANGE PAMLICO PASQUOTANK PENDER PERQUIMANS PERSON PITT POLK RANDOLPH RICHMOND ROBESON ROCKINGHAM ROWAN RUTHERFORD SAMPSON SCOTLAND STANLY STOKES SURRY SWAIN TRANSYLVANIA TYRRELL UNION VANCE WAKE WARREN WASHINGTON WATAUGA WAYNE WILKES WILSON YADKIN YANCEY`.split( + " ", + ), +); + +/** Parse position-sorted PDF text lines from NCSBE referendum reports. */ +export function parseReferendumLines( + lines: readonly string[], + sourceElectionDate?: string, +): NcsbeReferendumRecord[] { + let electionDate = sourceElectionDate ?? null; + let county: string | null = null; + let contest: string | null = null; + const output: NcsbeReferendumRecord[] = []; + + for (const raw of lines) { + const line = clean(raw).replace(/\s+/g, " "); + if (!line) continue; + const criteriaDate = /Election:\s*(\d{1,2}\/\d{1,2}\/\d{4})/i.exec( + line, + )?.[1]; + if (criteriaDate) electionDate = normalizeElectionDate(criteriaDate); + const headerCounty = /^([A-Z ]+) BOARD OF ELECTIONS$/ + .exec(line)?.[1] + ?.trim(); + if (headerCounty && NC_COUNTIES.has(headerCounty)) { + county = headerCounty; + contest = null; + continue; + } + if (NC_COUNTIES.has(line)) { + county = line; + contest = null; + continue; + } + if ( + /^(?:REFERENDUM CHOICES|CHOICE DESCRIPTION|CRITERIA:|Page \d+|\w{3} \d{1,2}, \d{4})/i.test( + line, + ) + ) { + continue; + } + const choice = /^(For|Against|Yes|No)\b[\s:.-]*(.*)$/i.exec(line); + if (choice && county && contest && electionDate) { + output.push({ + electionDate, + county, + contest, + choice: + choice[1]![0]!.toUpperCase() + choice[1]!.slice(1).toLowerCase(), + description: nullable(choice[2]), + }); + continue; + } + if (county && /^[A-Z0-9][A-Z0-9 '&().,/%-]+$/.test(line)) contest = line; + } + return output; +} + +/** Read the first text/CSV entry from a conventional NCSBE ZIP archive. */ +export function extractFirstTextFileFromZip(bytes: Uint8Array): string { + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); + let eocd = -1; + for ( + let i = bytes.length - 22; + i >= Math.max(0, bytes.length - 65_557); + i-- + ) { + if (view.getUint32(i, true) === 0x06054b50) { + eocd = i; + break; + } + } + if (eocd < 0) + throw new Error("ZIP end-of-central-directory record not found"); + const entries = view.getUint16(eocd + 10, true); + let offset = view.getUint32(eocd + 16, true); + + for (let entry = 0; entry < entries; entry++) { + if (view.getUint32(offset, true) !== 0x02014b50) + throw new Error("Invalid ZIP central directory"); + const method = view.getUint16(offset + 10, true); + const compressedSize = view.getUint32(offset + 20, true); + const fileNameLength = view.getUint16(offset + 28, true); + const extraLength = view.getUint16(offset + 30, true); + const commentLength = view.getUint16(offset + 32, true); + const localOffset = view.getUint32(offset + 42, true); + const fileName = new TextDecoder().decode( + bytes.subarray(offset + 46, offset + 46 + fileNameLength), + ); + offset += 46 + fileNameLength + extraLength + commentLength; + if (!/\.(?:txt|csv|tsv)$/i.test(fileName)) continue; + if (view.getUint32(localOffset, true) !== 0x04034b50) + throw new Error("Invalid ZIP local header"); + const localNameLength = view.getUint16(localOffset + 26, true); + const localExtraLength = view.getUint16(localOffset + 28, true); + const start = localOffset + 30 + localNameLength + localExtraLength; + const compressed = bytes.subarray(start, start + compressedSize); + const content = + method === 0 + ? compressed + : method === 8 + ? inflateRawSync(compressed) + : null; + if (!content) + throw new Error(`Unsupported ZIP compression method ${method}`); + return new TextDecoder().decode(content); + } + throw new Error("ZIP contains no text election-results file"); +} + +export function restrictToCycle( + rows: readonly T[], + cycleYear: number, +): T[] { + return rows.filter( + (row) => Number(row.electionDate.slice(0, 4)) === cycleYear, + ); +} diff --git a/apps/scraper/src/scrapers/ncsbe.config.ts b/apps/scraper/src/scrapers/ncsbe.config.ts new file mode 100644 index 00000000..4689c441 --- /dev/null +++ b/apps/scraper/src/scrapers/ncsbe.config.ts @@ -0,0 +1,11 @@ +import type { ScraperEnvContract } from "@acme/env"; + +export const ncsbeConfig = { + id: "ncsbe", + name: "North Carolina State Board of Elections", + source: "NCSBE candidate, referendum, and election-results files", + environment: { + required: ["POSTGRES_URL"], + optional: ["NCSBE_MAX_ITEMS"], + }, +} as const satisfies ScraperEnvContract; diff --git a/apps/scraper/src/scrapers/ncsbe.test.ts b/apps/scraper/src/scrapers/ncsbe.test.ts new file mode 100644 index 00000000..c07d555c --- /dev/null +++ b/apps/scraper/src/scrapers/ncsbe.test.ts @@ -0,0 +1,20 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { discoverCurrentCycleFiles } from "./ncsbe.js"; + +test("discovers structured current-cycle files without accepting history", () => { + const candidates = ` + 2026 Candidate List Spreadsheet (CSV) + 2024 Candidate List Spreadsheet (CSV) + 2026 Primary Referendum List`; + const results = ` + 2026 Mar 03 Election - Results (ZIP) + 2024 Nov 05 Election - Results (ZIP)`; + const files = discoverCurrentCycleFiles(candidates, results, 2026); + assert.deepEqual( + files.map((file) => file.kind), + ["candidates", "referenda", "results"], + ); + assert.ok(files.every((file) => file.url.includes("2026"))); +}); diff --git a/apps/scraper/src/scrapers/ncsbe.ts b/apps/scraper/src/scrapers/ncsbe.ts new file mode 100644 index 00000000..544d7d15 --- /dev/null +++ b/apps/scraper/src/scrapers/ncsbe.ts @@ -0,0 +1,382 @@ +/** + * Current-cycle NCSBE public election-data importer. + * + * Runtime discovery is intentionally limited to links for the current calendar + * year. Candidate contact/address fields and all voter-history products are out + * of scope and are never represented in the normalized parser output. + */ +import { createHash } from "node:crypto"; +import { load } from "cheerio"; +import { and, eq, gte, lt, or } from "drizzle-orm"; +import { getDocumentProxy } from "unpdf"; + +import { db } from "@acme/db/client"; +import { + ElectionCandidate, + ElectionReferendum, + ElectionResult, + ElectionSource, +} from "@acme/db/schema"; + +import type { Scraper } from "../utils/types.js"; +import type { + NcsbeCandidateRecord, + NcsbeReferendumRecord, + NcsbeResultRecord, +} from "./ncsbe-parsers.js"; +import { fetchWithRetry } from "../utils/fetch.js"; +import { createLogger } from "../utils/log.js"; +import { + extractFirstTextFileFromZip, + NCSBE_STRUCTURE_VERSION, + normalizeElectionDate, + parseCandidateCsv, + parseReferendumLines, + parseResultsTsv, + restrictToCycle, +} from "./ncsbe-parsers.js"; +import { ncsbeConfig } from "./ncsbe.config.js"; + +const logger = createLogger("ncsbe"); +const CANDIDATE_LIST_URL = "https://www.ncsbe.gov/results-data/candidate-lists"; +const RESULTS_LIST_URL = + "https://www.ncsbe.gov/results-data/election-results/historical-election-results-data"; +const USER_AGENT = "BillionCivicBot/1.0 (+https://billion.app)"; +const BATCH_SIZE = 750; + +type SourceKind = "candidates" | "referenda" | "results"; +type NcsbeRecord = + | NcsbeCandidateRecord + | NcsbeReferendumRecord + | NcsbeResultRecord; + +export interface NcsbeSourceFile { + kind: SourceKind; + url: string; + label: string; +} + +function currentCycleYear(now = new Date()): number { + return now.getUTCFullYear(); +} + +function absoluteUrl(href: string, base: string): string | null { + try { + return new URL(href, base).toString(); + } catch { + return null; + } +} + +function links(html: string, base: string): { label: string; url: string }[] { + const $ = load(html); + return $("a[href]") + .toArray() + .flatMap((anchor) => { + const url = absoluteUrl($(anchor).attr("href") ?? "", base); + return url + ? [{ label: $(anchor).text().replace(/\s+/g, " ").trim(), url }] + : []; + }); +} + +/** Discover only official files whose label/path identifies the current year. */ +export function discoverCurrentCycleFiles( + candidateHtml: string, + resultsHtml: string, + year = currentCycleYear(), +): NcsbeSourceFile[] { + const yearText = String(year); + const discovered: NcsbeSourceFile[] = []; + for (const link of links(candidateHtml, CANDIDATE_LIST_URL)) { + const haystack = `${link.label} ${decodeURIComponent(link.url)}`; + if (!haystack.includes(yearText)) continue; + if (/candidate/i.test(link.label) && /\.csv(?:$|\?)/i.test(link.url)) { + discovered.push({ kind: "candidates", ...link }); + } else if ( + /referendum/i.test(haystack) && + /\.pdf(?:$|\?)/i.test(link.url) + ) { + discovered.push({ kind: "referenda", ...link }); + } + } + for (const link of links(resultsHtml, RESULTS_LIST_URL)) { + const haystack = `${link.label} ${decodeURIComponent(link.url)}`; + if ( + haystack.includes(yearText) && + /results/i.test(link.label) && + /\.zip(?:$|\?)/i.test(link.url) && + !/precinct sort/i.test(link.label) + ) { + discovered.push({ kind: "results", ...link }); + } + } + return [ + ...new Map( + discovered.map((file) => [`${file.kind}:${file.url}`, file]), + ).values(), + ].sort((a, b) => a.kind.localeCompare(b.kind) || a.url.localeCompare(b.url)); +} + +async function fetchBytes(url: string): Promise { + const response = await fetchWithRetry(url, { + headers: { "User-Agent": USER_AGENT }, + maxRetries: 3, + timeoutMs: 45_000, + }); + return new Uint8Array(await response.arrayBuffer()); +} + +async function fetchPage(url: string): Promise { + return new TextDecoder().decode(await fetchBytes(url)); +} + +export async function extractPdfLines(bytes: Uint8Array): Promise { + const pdf = await getDocumentProxy(bytes); + const lines: string[] = []; + for (let pageNumber = 1; pageNumber <= pdf.numPages; pageNumber++) { + const page = await pdf.getPage(pageNumber); + const content = await page.getTextContent(); + const items = (content.items as unknown[]) + .flatMap((raw) => { + const item = raw as { str?: string; transform?: number[] }; + return typeof item.str === "string" && item.transform + ? [ + { + text: item.str.trim(), + x: item.transform[4] ?? 0, + y: item.transform[5] ?? 0, + }, + ] + : []; + }) + .filter((item) => item.text); + items.sort((a, b) => (Math.abs(a.y - b.y) > 2 ? b.y - a.y : a.x - b.x)); + let activeY: number | null = null; + let active: typeof items = []; + const flush = () => { + if (active.length) + lines.push( + active + .sort((a, b) => a.x - b.x) + .map((item) => item.text) + .join(" "), + ); + active = []; + }; + for (const item of items) { + if (activeY !== null && Math.abs(activeY - item.y) > 2) flush(); + activeY = item.y; + active.push(item); + } + flush(); + } + return lines; +} + +export async function parseReferendumPdf( + bytes: Uint8Array, + sourceUrl: string, +): Promise { + return parseReferendumLines( + await extractPdfLines(bytes), + sourceDateFromUrl(sourceUrl) ?? undefined, + ); +} + +function sha256(bytes: Uint8Array): string { + return createHash("sha256").update(bytes).digest("hex"); +} + +function sourceDateFromUrl(url: string): string | null { + const compact = /(?:_|referendums_)(20\d{2})(\d{2})(\d{2})/.exec(url); + return compact + ? normalizeElectionDate(`${compact[2]}/${compact[3]}/${compact[1]}`) + : null; +} + +function groupsByDate( + rows: readonly T[], +): Map { + const groups = new Map(); + for (const row of rows) + groups.set(row.electionDate, [ + ...(groups.get(row.electionDate) ?? []), + row, + ]); + return groups; +} + +async function insertBatches( + rows: readonly T[], + insert: (batch: T[]) => Promise, +): Promise { + for (let start = 0; start < rows.length; start += BATCH_SIZE) { + await insert(rows.slice(start, start + BATCH_SIZE)); + } +} + +async function persistDateGroup( + file: NcsbeSourceFile, + electionDate: string, + checksum: string, + fetchedAt: Date, + rows: NcsbeRecord[], +): Promise { + await db.transaction(async (tx) => { + const [cached] = await tx + .select({ + id: ElectionSource.id, + checksum: ElectionSource.checksum, + structureVersion: ElectionSource.structureVersion, + }) + .from(ElectionSource) + .where( + and( + eq(ElectionSource.provider, "ncsbe"), + eq(ElectionSource.sourceKind, file.kind), + eq(ElectionSource.electionDate, electionDate), + eq(ElectionSource.sourceUrl, file.url), + ), + ) + .limit(1); + if ( + cached?.checksum === checksum && + cached.structureVersion === NCSBE_STRUCTURE_VERSION + ) { + await tx + .update(ElectionSource) + .set({ fetchedAt }) + .where(eq(ElectionSource.id, cached.id)); + return; + } + + const [source] = await tx + .insert(ElectionSource) + .values({ + provider: "ncsbe", + sourceKind: file.kind, + electionDate, + sourceUrl: file.url, + checksum, + structureVersion: NCSBE_STRUCTURE_VERSION, + certificationStatus: + file.kind === "results" ? "official_not_certified" : "not_applicable", + fetchedAt, + }) + .onConflictDoUpdate({ + target: [ + ElectionSource.provider, + ElectionSource.sourceKind, + ElectionSource.electionDate, + ElectionSource.sourceUrl, + ], + set: { checksum, structureVersion: NCSBE_STRUCTURE_VERSION, fetchedAt }, + }) + .returning({ id: ElectionSource.id }); + if (!source) throw new Error(`Unable to upsert source ${file.url}`); + + if (file.kind === "candidates") { + await tx + .delete(ElectionCandidate) + .where(eq(ElectionCandidate.sourceId, source.id)); + const candidates = rows as NcsbeCandidateRecord[]; + await insertBatches(candidates, (batch) => + tx + .insert(ElectionCandidate) + .values(batch.map((row) => ({ ...row, sourceId: source.id }))), + ); + } else if (file.kind === "referenda") { + await tx + .delete(ElectionReferendum) + .where(eq(ElectionReferendum.sourceId, source.id)); + const referenda = rows as NcsbeReferendumRecord[]; + await insertBatches(referenda, (batch) => + tx + .insert(ElectionReferendum) + .values(batch.map((row) => ({ ...row, sourceId: source.id }))), + ); + } else { + await tx + .delete(ElectionResult) + .where(eq(ElectionResult.sourceId, source.id)); + const results = rows as NcsbeResultRecord[]; + await insertBatches(results, (batch) => + tx + .insert(ElectionResult) + .values(batch.map((row) => ({ ...row, sourceId: source.id }))), + ); + } + }); +} + +async function importFile( + file: NcsbeSourceFile, + year: number, +): Promise { + const fetchedAt = new Date(); + const bytes = await fetchBytes(file.url); + const checksum = sha256(bytes); + let rows: NcsbeRecord[]; + if (file.kind === "candidates") { + rows = parseCandidateCsv(new TextDecoder().decode(bytes)); + } else if (file.kind === "results") { + rows = parseResultsTsv(extractFirstTextFileFromZip(bytes)); + } else { + rows = await parseReferendumPdf(bytes, file.url); + } + rows = restrictToCycle(rows, year); + if (rows.length === 0) + throw new Error(`${file.url} produced no current-cycle records`); + for (const [electionDate, dateRows] of groupsByDate(rows)) { + await persistDateGroup(file, electionDate, checksum, fetchedAt, dateRows); + } + return rows.length; +} + +export async function scrapeNcsbe( + maxItems = 4, + now = new Date(), +): Promise { + const year = currentCycleYear(now); + logger.info(`Discovering NCSBE public election files for the ${year} cycle…`); + const [candidateHtml, resultsHtml] = await Promise.all([ + fetchPage(CANDIDATE_LIST_URL), + fetchPage(RESULTS_LIST_URL), + ]); + const files = discoverCurrentCycleFiles( + candidateHtml, + resultsHtml, + year, + ).slice(0, maxItems); + if (files.length === 0) + throw new Error(`No NCSBE files discovered for ${year}`); + for (const file of files) { + const count = await importFile(file, year); + logger.success(`${file.kind}: persisted ${count} records from ${file.url}`); + } + // Keep persistence bounded to the product's current-cycle scope. Cascading + // foreign keys remove normalized rows only after every current import succeeds. + const stale = await db + .delete(ElectionSource) + .where( + and( + eq(ElectionSource.provider, "ncsbe"), + or( + lt(ElectionSource.electionDate, `${year}-01-01`), + gte(ElectionSource.electionDate, `${year + 1}-01-01`), + ), + ), + ) + .returning({ id: ElectionSource.id }); + if (stale.length) + logger.info(`Removed ${stale.length} out-of-cycle NCSBE source snapshots.`); +} + +export const ncsbe: Scraper = { + ...ncsbeConfig, + scrape: (options) => + scrapeNcsbe( + (options?.maxItems ?? Number(process.env.NCSBE_MAX_ITEMS)) || 4, + ), +}; diff --git a/apps/scraper/src/scrapers/texas-current-election.config.ts b/apps/scraper/src/scrapers/texas-current-election.config.ts new file mode 100644 index 00000000..4366309a --- /dev/null +++ b/apps/scraper/src/scrapers/texas-current-election.config.ts @@ -0,0 +1,11 @@ +import type { ScraperEnvContract } from "@acme/env"; + +export const texasCurrentElectionConfig = { + id: "texas-current-election", + name: "Texas SOS/TLC current election cycle", + source: "Texas Secretary of State and Texas Legislative Council", + environment: { + required: ["POSTGRES_URL"], + optional: ["TX_SOS_MAX_ITEMS"], + }, +} as const satisfies ScraperEnvContract; diff --git a/apps/scraper/src/scrapers/texas-current-election.test.ts b/apps/scraper/src/scrapers/texas-current-election.test.ts new file mode 100644 index 00000000..0c546f17 --- /dev/null +++ b/apps/scraper/src/scrapers/texas-current-election.test.ts @@ -0,0 +1,17 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { discoverLatestTlcAnalysis } from "./texas-current-election.js"; + +void test("TLC discovery chooses the newest full report and skips condensed PDFs", () => { + const html = ` + November 7, 2023 + Condensed + 2025 + `; + assert.deepEqual(discoverLatestTlcAnalysis(html), { + year: 2025, + title: "2025", + url: "https://tlc.texas.gov/docs/amendments/analyses25.pdf", + }); +}); diff --git a/apps/scraper/src/scrapers/texas-current-election.ts b/apps/scraper/src/scrapers/texas-current-election.ts new file mode 100644 index 00000000..6a2dc07b --- /dev/null +++ b/apps/scraper/src/scrapers/texas-current-election.ts @@ -0,0 +1,285 @@ +/** + * Current-cycle Texas statewide election ingestion. + * + * The SOS Civix application publishes base64-wrapped JSON for election + * discovery, contests/results, reporting status, and county totals. TLC's + * publications page links the latest cycle-specific constitutional-amendment + * analysis PDF. This scraper discovers both rather than constructing year URLs, + * parses them deterministically, and stores separate provider snapshots. + */ + +import { createHash } from "node:crypto"; +import * as cheerio from "cheerio"; +import { getDocumentProxy } from "unpdf"; + +import type { + TexasElectionDefinition, + TexasSosElection, + TexasSosSnapshotData, + TexasTlcSnapshotData, + TlcTextPage, +} from "@acme/api/lib/texas-election-data"; +import { + parseTexasSosDiscovery, + parseTexasSosElection, + parseTexasTlcAnalysis, + TEXAS_CURRENT_SCOPE, + TEXAS_RESULTS_URL, + TEXAS_SOS_PROVIDER, + TEXAS_TLC_PROVIDER, + TEXAS_TLC_PUBLICATIONS_URL, +} from "@acme/api/lib/texas-election-data"; +import { db } from "@acme/db/client"; +import { ElectionSourceSnapshot } from "@acme/db/schema"; + +import type { Scraper } from "../utils/types.js"; +import { fetchWithRetry } from "../utils/fetch.js"; +import { createLogger } from "../utils/log.js"; +import { texasCurrentElectionConfig } from "./texas-current-election.config.js"; + +const logger = createLogger("texas-current-election"); +const API_BASE = + "https://goelect.txelections.civixapps.com/api-ivis-system/api/s3/enr"; +const USER_AGENT = + "Mozilla/5.0 (compatible; BillionCivicBot/1.0; +https://billion.app)"; + +interface TlcPublication { + year: number; + title: string; + url: string; +} + +function sha256(value: string | Uint8Array): string { + return createHash("sha256").update(value).digest("hex"); +} + +async function fetchJson(url: string): Promise { + const response = await fetchWithRetry(url, { + headers: { Accept: "application/json", "User-Agent": USER_AGENT }, + }); + return response.json() as Promise; +} + +async function fetchText(url: string): Promise { + const response = await fetchWithRetry(url, { + headers: { Accept: "text/html", "User-Agent": USER_AGENT }, + }); + return response.text(); +} + +async function fetchBytes(url: string): Promise { + const response = await fetchWithRetry(url, { + headers: { Accept: "application/pdf", "User-Agent": USER_AGENT }, + timeoutMs: 60_000, + }); + return new Uint8Array(await response.arrayBuffer()); +} + +/** Discover the newest full TLC analysis PDF from the publications index. */ +export function discoverLatestTlcAnalysis( + html: string, + baseUrl = TEXAS_TLC_PUBLICATIONS_URL, +): TlcPublication | null { + const $ = cheerio.load(html); + const candidates: TlcPublication[] = []; + $("a[href]").each((_index, element) => { + const href = $(element).attr("href"); + if (!href || /condensed/i.test(href)) return; + const match = /\/analyses(\d{2}|\d{4})\.pdf(?:$|[?#])/i.exec(href); + if (!match?.[1]) return; + const short = Number.parseInt(match[1], 10); + const year = short < 100 ? 2000 + short : short; + if (!Number.isInteger(year) || year > new Date().getFullYear()) return; + candidates.push({ + year, + title: + $(element).text().replace(/\s+/g, " ").trim() || + `Analyses of Proposed Constitutional Amendments (${year})`, + url: new URL(href, baseUrl).toString(), + }); + }); + return candidates.sort((a, b) => b.year - a.year)[0] ?? null; +} + +/** Extract each PDF page independently so citations retain real page numbers. */ +export async function extractTlcPages( + bytes: Uint8Array, +): Promise { + const pdf = await getDocumentProxy(bytes); + const pages: TlcTextPage[] = []; + for (let pageNumber = 1; pageNumber <= pdf.numPages; pageNumber++) { + const page = await pdf.getPage(pageNumber); + const content = await page.getTextContent(); + const text = (content.items as Array<{ str?: string }>) + .map((item) => item.str?.trim() ?? "") + .filter(Boolean) + .join(" "); + pages.push({ page: pageNumber, text }); + } + return pages; +} + +async function loadSosElection( + definition: TexasElectionDefinition, +): Promise { + try { + const [election, counties] = await Promise.all([ + fetchJson(`${API_BASE}/election/${definition.id}`), + fetchJson(`${API_BASE}/election/countyInfo/${definition.id}`).catch( + () => undefined, + ), + ]); + return parseTexasSosElection(election, definition, counties); + } catch (error) { + logger.warn(`SOS election ${definition.id} could not be parsed:`, error); + return null; + } +} + +async function upsertSnapshot(input: { + cycleYear: number; + provider: string; + sourceVersion: string; + data: Record; + diagnostics: string[]; + sourceUrls: string[]; +}): Promise { + const contentHash = sha256(JSON.stringify(input.data)); + const now = new Date(); + await db + .insert(ElectionSourceSnapshot) + .values({ + jurisdiction: "TX", + cycleYear: input.cycleYear, + provider: input.provider, + scope: TEXAS_CURRENT_SCOPE, + sourceVersion: input.sourceVersion, + contentHash, + data: input.data, + diagnostics: input.diagnostics, + sourceUrls: input.sourceUrls, + fetchedAt: now, + }) + .onConflictDoUpdate({ + target: [ + ElectionSourceSnapshot.jurisdiction, + ElectionSourceSnapshot.cycleYear, + ElectionSourceSnapshot.provider, + ElectionSourceSnapshot.scope, + ], + set: { + sourceVersion: input.sourceVersion, + contentHash, + data: input.data, + diagnostics: input.diagnostics, + sourceUrls: input.sourceUrls, + fetchedAt: now, + }, + }); +} + +function selectedDefinitions( + definitions: TexasElectionDefinition[], + amendmentOnly: boolean, + remaining: number, +): TexasElectionDefinition[] { + const filtered = amendmentOnly + ? definitions.filter((definition) => + /constitutional amendment/i.test(definition.name), + ) + : definitions; + return filtered.slice(0, Math.max(0, remaining)); +} + +async function scrape(maxItems = 12): Promise { + logger.info("Discovering current Texas SOS elections and TLC analysis…"); + const [constantsRaw, publicationsHtml] = await Promise.all([ + fetchJson(`${API_BASE}/electionConstants`), + fetchText(TEXAS_TLC_PUBLICATIONS_URL), + ]); + const discovery = parseTexasSosDiscovery(constantsRaw); + const tlcPublication = discoverLatestTlcAnalysis(publicationsHtml); + const diagnostics: string[] = []; + if (!tlcPublication) + diagnostics.push("No current TLC amendment analysis PDF discovered"); + + let processed = 0; + const years = new Set([discovery.cycleYear]); + if (tlcPublication) years.add(tlcPublication.year); + for (const year of [...years].sort((a, b) => b - a)) { + const definitions = selectedDefinitions( + discovery.electionsByYear[year] ?? [], + year !== discovery.cycleYear, + maxItems - processed, + ); + const elections: TexasSosElection[] = []; + for (const definition of definitions) { + const election = await loadSosElection(definition); + processed++; + if (election) elections.push(election); + } + if (!elections.length) { + diagnostics.push(`No SOS election payloads parsed for ${year}`); + continue; + } + const data: TexasSosSnapshotData = { cycleYear: year, elections }; + await upsertSnapshot({ + cycleYear: year, + provider: TEXAS_SOS_PROVIDER, + sourceVersion: elections + .map((election) => election.sourceVersion) + .join(","), + data: data as unknown as Record, + diagnostics: diagnostics.filter((item) => item.includes(String(year))), + sourceUrls: [TEXAS_RESULTS_URL], + }); + logger.success( + `Texas SOS ${year}: persisted ${elections.length} election(s).`, + ); + } + + if (tlcPublication && processed < maxItems) { + try { + const bytes = await fetchBytes(tlcPublication.url); + const pages = await extractTlcPages(bytes); + const parsed: TexasTlcSnapshotData = parseTexasTlcAnalysis( + pages, + tlcPublication.url, + ); + parsed.publicationTitle = tlcPublication.title; + const tlcDiagnostics = [ + ...diagnostics.filter((item) => item.includes("TLC")), + ...parsed.measures.flatMap((measure) => + measure.diagnostics.map( + (item) => `Proposition ${measure.propositionNumber}: ${item}`, + ), + ), + ]; + if (!parsed.measures.length) { + tlcDiagnostics.push("TLC PDF contained no parseable propositions"); + } + await upsertSnapshot({ + cycleYear: parsed.cycleYear, + provider: TEXAS_TLC_PROVIDER, + sourceVersion: `tlc:${parsed.cycleYear}:${sha256(bytes).slice(0, 16)}`, + data: parsed as unknown as Record, + diagnostics: tlcDiagnostics, + sourceUrls: [TEXAS_TLC_PUBLICATIONS_URL, tlcPublication.url], + }); + logger.success( + `Texas TLC ${parsed.cycleYear}: persisted ${parsed.measures.length} proposition analyses.`, + ); + } catch (error) { + logger.warn( + "TLC analysis could not be parsed; SOS data remains available:", + error, + ); + } + } +} + +export const texasCurrentElection: Scraper = { + ...texasCurrentElectionConfig, + scrape: (options) => + scrape((options?.maxItems ?? Number(process.env.TX_SOS_MAX_ITEMS)) || 12), +}; diff --git a/apps/scraper/src/scrapers/texas-legislature-parser.ts b/apps/scraper/src/scrapers/texas-legislature-parser.ts new file mode 100644 index 00000000..b7f63569 --- /dev/null +++ b/apps/scraper/src/scrapers/texas-legislature-parser.ts @@ -0,0 +1,344 @@ +import { load } from "cheerio"; + +export const TEXAS_JURISDICTION = + "ocd-jurisdiction/country:us/state:tx/government"; + +export interface TexasDocument { + type: "bill_text" | "analysis" | "fiscal_note"; + description: string; + htmlUrl?: string; + pdfUrl?: string; + ftpHtmlUrl?: string; + ftpPdfUrl?: string; + text?: string; +} + +export interface TexasVote { + identifier: string; + date?: string; + chamber?: "House" | "Senate"; + motion?: string; + result?: string; + sourceUrl?: string; + counts: { option: string; value: number }[]; + votes: { option: string; voterName: string; openStatesId?: string }[]; +} + +export interface ParsedTexasBill { + billNumber: string; + title: string; + sponsor?: string; + status?: string; + introducedDate?: Date; + chamber: "House" | "Senate"; + jurisdiction: typeof TEXAS_JURISDICTION; + legislativeSession: string; + subjects: string[]; + sponsorships: { + name: string; + classification: "primary" | "cosponsor"; + chamber: "House" | "Senate"; + }[]; + documents: TexasDocument[]; + votes: TexasVote[]; + actions: { date: string; text: string; type?: string }[]; + url: string; +} + +function normalizeSpace(value: string | undefined): string | undefined { + const normalized = value?.replace(/\s+/g, " ").trim(); + return normalized || undefined; +} + +function parseTexasDate(value: string | undefined): string | undefined { + const text = normalizeSpace(value); + if (!text) return undefined; + const match = /^(\d{1,2})\/(\d{1,2})\/(\d{4})$/.exec(text); + if (match) { + const [, month, day, year] = match; + return `${year}-${month!.padStart(2, "0")}-${day!.padStart(2, "0")}`; + } + const parsed = new Date(text); + return Number.isNaN(parsed.valueOf()) + ? undefined + : parsed.toISOString().slice(0, 10); +} + +export function normalizeTexasBillNumber(value: string): string { + const match = /(HCR|HJR|HR|HB|SCR|SJR|SR|SB)\s*0*(\d+)/i.exec(value); + if (!match) throw new Error(`Unrecognized Texas bill identifier: ${value}`); + return `${match[1]!.toUpperCase()} ${Number(match[2])}`; +} + +function chamberFor(identifier: string): "House" | "Senate" { + return identifier.startsWith("H") ? "House" : "Senate"; +} + +function splitNames(value: string | undefined): string[] { + return (value ?? "") + .split("|") + .map((name) => normalizeSpace(name)) + .filter((name): name is string => Boolean(name)); +} + +function firstText( + $: ReturnType, + selectors: readonly string[], +): string | undefined { + for (const selector of selectors) { + const value = normalizeSpace($(selector).first().text()); + if (value) return value; + } + return undefined; +} + +function parseDocuments($: ReturnType): TexasDocument[] { + const documents: TexasDocument[] = []; + const groups = [ + ["billtext > docTypes > bill > versions > version", "bill_text"], + ["billtext > docTypes > analysis > versions > version", "analysis"], + ["billtext > docTypes > fiscalNote > versions > version", "fiscal_note"], + ] as const; + + for (const [selector, type] of groups) { + $(selector).each((_, element) => { + const version = $(element); + const description = + normalizeSpace(version.find("versionDescription").first().text()) ?? + type.replace("_", " "); + const htmlUrl = normalizeSpace(version.find("WebHTMLURL").first().text()); + const pdfUrl = normalizeSpace(version.find("WebPDFURL").first().text()); + const ftpHtmlUrl = normalizeSpace( + version.find("FTPHTMLURL").first().text(), + ); + const ftpPdfUrl = normalizeSpace( + version.find("FTPPDFURL").first().text(), + ); + if (htmlUrl || pdfUrl || ftpHtmlUrl || ftpPdfUrl) { + documents.push({ + type, + description, + ...(htmlUrl && { htmlUrl }), + ...(pdfUrl && { pdfUrl }), + ...(ftpHtmlUrl && { ftpHtmlUrl }), + ...(ftpPdfUrl && { ftpPdfUrl }), + }); + } + }); + } + return documents; +} + +function parseVotes($: ReturnType): TexasVote[] { + const votes: TexasVote[] = []; + $("votes > vote, recordVotes > vote, voteHistory > vote").each( + (_, element) => { + const vote = $(element); + const text = (selectors: string[]) => { + for (const selector of selectors) { + const value = normalizeSpace(vote.find(selector).first().text()); + if (value) return value; + } + return undefined; + }; + const identifier = text([ + "identifier", + "voteNumber", + "rollCallId", + "actionNumber", + ]); + if (!identifier) return; + const chamberText = text(["chamber", "organization"]); + const chamber = chamberText + ? chamberText.toLowerCase().startsWith("h") + ? "House" + : chamberText.toLowerCase().startsWith("s") + ? "Senate" + : undefined + : undefined; + const counts: TexasVote["counts"] = []; + vote.find("counts > count, totals > total").each((__, countElement) => { + const count = $(countElement); + const option = normalizeSpace( + count.attr("option") ?? count.find("option").first().text(), + ); + const value = Number( + normalizeSpace( + count.attr("value") ?? count.find("value").first().text(), + ), + ); + if (option && Number.isInteger(value)) counts.push({ option, value }); + }); + const memberVotes: TexasVote["votes"] = []; + vote + .find("voters > voter, memberVotes > vote") + .each((__, voterElement) => { + const voter = $(voterElement); + const voterName = normalizeSpace( + voter.attr("name") ?? + voter.find("name, voterName, memberName").first().text(), + ); + const option = normalizeSpace( + voter.attr("option") ?? voter.find("option, vote").first().text(), + ); + if (voterName && option) memberVotes.push({ voterName, option }); + }); + votes.push({ + identifier, + ...(parseTexasDate(text(["date", "voteDate"])) && { + date: parseTexasDate(text(["date", "voteDate"])), + }), + ...(chamber && { chamber }), + ...(text(["motion", "motionText", "description"]) && { + motion: text(["motion", "motionText", "description"]), + }), + ...(text(["result", "outcome"]) && { + result: text(["result", "outcome"]), + }), + ...(text(["sourceUrl", "url"]) && { + sourceUrl: text(["sourceUrl", "url"]), + }), + counts, + votes: memberVotes, + }); + }, + ); + $("actions > action").each((_, element) => { + const action = $(element); + if ( + normalizeSpace(action.find("description").first().text()) !== + "Record vote" + ) { + return; + } + const identifier = normalizeSpace(action.find("comment").first().text()); + if (!identifier) return; + const actionNumber = normalizeSpace( + action.find("actionNumber").first().text(), + ); + votes.push({ + identifier, + ...(parseTexasDate(action.find("date").first().text()) && { + date: parseTexasDate(action.find("date").first().text()), + }), + ...(actionNumber?.startsWith("H") && { chamber: "House" }), + ...(actionNumber?.startsWith("S") && { chamber: "Senate" }), + motion: "Record vote", + counts: [], + votes: [], + }); + }); + $("committees > house, committees > senate").each((_, element) => { + const committee = $(element); + const name = normalizeSpace(committee.attr("name")); + if (!name) return; + const chamber = + element.tagName.toLowerCase() === "house" ? "House" : "Senate"; + const countAttributes = [ + ["Yea", "ayeVotes"], + ["Nay", "nayVotes"], + ["Present not voting", "presentNotVotingVotes"], + ["Absent", "absentVotes"], + ] as const; + const counts = countAttributes.flatMap(([option, attribute]) => { + const value = Number(committee.attr(attribute)); + return Number.isInteger(value) ? [{ option, value }] : []; + }); + if (counts.length === 0) return; + votes.push({ + identifier: `committee:${chamber.toLowerCase()}:${name}`, + chamber, + motion: `${name} committee vote`, + result: normalizeSpace(committee.attr("status")), + counts, + votes: [], + }); + }); + return votes; +} + +export function parseTexasBillHistory( + xml: string, + session: string, +): ParsedTexasBill { + const $ = load(xml, { xml: true }); + const root = $.root().children().first(); + const rawIdentifier = root.attr("bill") ?? firstText($, ["billNumber"]); + if (!rawIdentifier) + throw new Error("Texas bill history is missing bill identity"); + const billNumber = normalizeTexasBillNumber(rawIdentifier); + const title = firstText($, ["caption"]); + if (!title || title.includes("Bill does not exist")) { + throw new Error(`${billNumber} does not contain a usable caption`); + } + const chamber = chamberFor(billNumber); + const otherChamber = chamber === "House" ? "Senate" : "House"; + const sponsorships: ParsedTexasBill["sponsorships"] = []; + for (const [selector, classification, sponsorChamber] of [ + ["authors", "primary", chamber], + ["coauthors", "cosponsor", chamber], + ["sponsors", "primary", otherChamber], + ["cosponsors", "cosponsor", otherChamber], + ] as const) { + for (const name of splitNames($(selector).first().text())) { + sponsorships.push({ name, classification, chamber: sponsorChamber }); + } + } + + const actions: ParsedTexasBill["actions"] = []; + $("actions > action").each((_, element) => { + const action = $(element); + const date = parseTexasDate(action.find("date").first().text()); + const text = normalizeSpace(action.find("description").first().text()); + if (!date || !text) return; + const actionNumber = normalizeSpace( + action.find("actionNumber").first().text(), + ); + actions.push({ date, text, ...(actionNumber && { type: actionNumber }) }); + }); + actions.sort((left, right) => left.date.localeCompare(right.date)); + + const sponsor = sponsorships + .filter((item) => item.classification === "primary") + .map((item) => item.name) + .join(" | ") + .slice(0, 256); + const billSlug = billNumber.replace(/\s+/g, ""); + return { + billNumber, + title, + ...(sponsor && { sponsor }), + ...(actions.at(-1)?.text && { status: actions.at(-1)!.text.slice(0, 100) }), + ...(actions[0]?.date && { + introducedDate: new Date(`${actions[0].date}T12:00:00.000Z`), + }), + chamber, + jurisdiction: TEXAS_JURISDICTION, + legislativeSession: session.toUpperCase(), + subjects: $("subjects > subject") + .map((_, element) => normalizeSpace($(element).text())) + .get() + .filter((value): value is string => Boolean(value)), + sponsorships, + documents: parseDocuments($), + votes: parseVotes($), + actions, + url: `https://capitol.texas.gov/BillLookup/History.aspx?LegSess=${encodeURIComponent(session.toUpperCase())}&Bill=${billSlug}`, + }; +} + +export function htmlToText(html: string): string { + const $ = load(html); + $("script, style, nav, header, footer").remove(); + $("br, p, div, h1, h2, h3, h4, h5, h6, li, tr").each((_, element) => { + $(element).append(" "); + }); + return $.root().text().replace(/\s+/g, " ").trim(); +} + +export function openStatesSessionName(session: string): string { + const normalized = session.toUpperCase(); + if (/^\d{2}R$/.test(normalized)) return normalized.slice(0, 2); + if (/^\d{3}$/.test(normalized)) return normalized; + throw new Error(`Invalid Texas legislative session: ${session}`); +} diff --git a/apps/scraper/src/scrapers/texas-legislature-source.ts b/apps/scraper/src/scrapers/texas-legislature-source.ts new file mode 100644 index 00000000..4eba3d35 --- /dev/null +++ b/apps/scraper/src/scrapers/texas-legislature-source.ts @@ -0,0 +1,134 @@ +import { Writable } from "node:stream"; + +import { Client } from "basic-ftp"; + +import type { TexasDocument } from "./texas-legislature-parser.js"; + +export interface BulkEntry { + name: string; + isDirectory: boolean; +} + +export interface TexasBulkClient { + list(path: string): Promise; + download(path: string): Promise; + close(): void; +} + +export class TexasFtpClient implements TexasBulkClient { + private readonly client = new Client(30_000); + + async connect(): Promise { + await this.client.access({ + host: "ftp.legis.state.tx.us", + user: "anonymous", + password: "billion@example.invalid", + secure: false, + }); + } + + async list(path: string): Promise { + return (await this.client.list(path)).map((entry) => ({ + name: entry.name, + isDirectory: entry.isDirectory, + })); + } + + async download(path: string): Promise { + const chunks: Buffer[] = []; + const sink = new Writable({ + write(chunk: Buffer | string, _encoding, callback) { + chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); + callback(); + }, + }); + await this.client.downloadTo(sink, path); + return Buffer.concat(chunks); + } + + close(): void { + this.client.close(); + } +} + +export function selectCurrentTexasSession(names: readonly string[]): string { + const sessions = names + .map((name) => name.toUpperCase()) + .filter((name) => /^\d{2}(?:R|\d)$/.test(name)) + .sort((left, right) => { + const legislature = Number(left.slice(0, 2)) - Number(right.slice(0, 2)); + if (legislature !== 0) return legislature; + const rank = (value: string) => + value[2] === "R" ? 0 : Number(value[2]) || 0; + return rank(left) - rank(right); + }); + const session = sessions.at(-1); + if (!session) throw new Error("No Texas legislative sessions found in /bills"); + return session; +} + +export async function listFilesRecursively( + client: TexasBulkClient, + root: string, +): Promise { + const entries = (await client.list(root)).sort((a, b) => + a.name.localeCompare(b.name), + ); + const files: string[] = []; + for (const entry of entries) { + if (entry.name === "." || entry.name === "..") continue; + const child = `${root.replace(/\/$/, "")}/${entry.name}`; + if (entry.isDirectory) files.push(...(await listFilesRecursively(client, child))); + else files.push(child); + } + return files; +} + +const folderByPrefix: Record = { + HB: "house_bills", + HCR: "house_concurrent_resolutions", + HJR: "house_joint_resolutions", + HR: "house_resolutions", + SB: "senate_bills", + SCR: "senate_concurrent_resolutions", + SJR: "senate_joint_resolutions", + SR: "senate_resolutions", +}; + +const rootByType: Record = { + bill_text: "billtext", + analysis: "analysis", + fiscal_note: "fiscalnotes", +}; + +export function bulkHtmlPath( + session: string, + document: Pick, +): string | undefined { + if (document.ftpHtmlUrl) { + try { + return decodeURIComponent(new URL(document.ftpHtmlUrl).pathname); + } catch { + return undefined; + } + } + if (!document.htmlUrl) return undefined; + let filename: string; + try { + filename = new URL(document.htmlUrl).pathname.split("/").at(-1) ?? ""; + } catch { + return undefined; + } + const match = /^(HCR|HJR|HR|HB|SCR|SJR|SR|SB)(\d{5})[A-Z0-9]*\.html?$/i.exec( + filename, + ); + if (!match) return undefined; + const prefix = match[1]!.toUpperCase(); + const number = Number(match[2]); + const start = number < 100 ? 1 : Math.floor(number / 100) * 100; + const end = number < 100 ? 99 : start + 99; + const pad = (value: number) => String(value).padStart(5, "0"); + const documentRoot = + document.type === "fiscal_note" ? "fiscalNotes" : rootByType[document.type]; + return `/bills/${session.toUpperCase()}/${documentRoot}/HTML/${folderByPrefix[prefix]}/${prefix}${pad(start)}_${prefix}${pad(end)}/${filename}`; +} diff --git a/apps/scraper/src/scrapers/texas-legislature.config.ts b/apps/scraper/src/scrapers/texas-legislature.config.ts new file mode 100644 index 00000000..8e4be629 --- /dev/null +++ b/apps/scraper/src/scrapers/texas-legislature.config.ts @@ -0,0 +1,16 @@ +import type { ScraperEnvContract } from "@acme/env"; + +export const texasLegislatureConfig = { + id: "texas-legislature", + name: "Texas Legislature Online", + source: + "Texas Legislative Council anonymous FTP — current-session history XML and bulk documents", + environment: { + required: ["POSTGRES_URL"], + optional: [ + "OPEN_STATES_API_KEY", + "TEXAS_LEGISLATURE_SESSION", + "TEXAS_LEGISLATURE_MAX_ITEMS", + ], + }, +} as const satisfies ScraperEnvContract; diff --git a/apps/scraper/src/scrapers/texas-legislature.test.ts b/apps/scraper/src/scrapers/texas-legislature.test.ts new file mode 100644 index 00000000..e51d886c --- /dev/null +++ b/apps/scraper/src/scrapers/texas-legislature.test.ts @@ -0,0 +1,105 @@ +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import test from "node:test"; + +import { CreateBillSchema } from "@acme/db/schema"; + +import { + htmlToText, + openStatesSessionName, + parseTexasBillHistory, +} from "./texas-legislature-parser.js"; +import { + bulkHtmlPath, + listFilesRecursively, + selectCurrentTexasSession, + type TexasBulkClient, +} from "./texas-legislature-source.js"; + +const fixture = await readFile( + new URL("./fixtures/texas-hb9-history.xml", import.meta.url), + "utf8", +); + +void test("parses official-style Texas bill history deterministically", () => { + const bill = parseTexasBillHistory(fixture, "89R"); + assert.equal(bill.billNumber, "HB 9"); + assert.equal(bill.chamber, "House"); + assert.equal(bill.legislativeSession, "89R"); + assert.equal(bill.sponsor, "Meyer | Bonnen | Bettencourt"); + assert.equal(bill.status, "Effective immediately"); + assert.equal(bill.introducedDate?.toISOString(), "2024-11-12T12:00:00.000Z"); + assert.deepEqual( + bill.documents.map((document) => document.type), + ["bill_text", "bill_text", "analysis", "fiscal_note"], + ); + assert.deepEqual(bill.votes[0], { + identifier: "RV#2999", + date: "2025-05-19", + chamber: "House", + motion: "Record vote", + counts: [], + votes: [], + }); + assert.deepEqual(bill.votes[1]?.counts, [ + { option: "Yea", value: 9 }, + { option: "Nay", value: 1 }, + { option: "Present not voting", value: 0 }, + { option: "Absent", value: 1 }, + ]); +}); + +void test("maps public document names to the official FTP bulk tree", () => { + const bill = parseTexasBillHistory(fixture, "89R"); + assert.equal( + bulkHtmlPath("89R", bill.documents[0]!), + "/bills/89R/billtext/HTML/house_bills/HB00001_HB00099/HB00009I.HTM", + ); + assert.equal( + bulkHtmlPath("89R", bill.documents.at(-1)!), + "/bills/89R/fiscalNotes/HTML/house_bills/HB00001_HB00099/HB00009I.HTM", + ); +}); + +void test("selects only the latest Texas session and maps Open States names", () => { + assert.equal(selectCurrentTexasSession(["88R", "89R", "891", "892"]), "892"); + assert.equal(openStatesSessionName("89R"), "89"); + assert.equal(openStatesSessionName("892"), "892"); +}); + +void test("walks bulk directories in stable lexical order", async () => { + const listings: Record = { + "/root": [ + { name: "z.xml", isDirectory: false }, + { name: "a", isDirectory: true }, + ], + "/root/a": [{ name: "b.xml", isDirectory: false }], + }; + const client: TexasBulkClient = { + list: async (path) => listings[path] ?? [], + download: async () => Buffer.alloc(0), + close: () => undefined, + }; + assert.deepEqual(await listFilesRecursively(client, "/root"), [ + "/root/a/b.xml", + "/root/z.xml", + ]); +}); + +void test("extracts readable deterministic text from bulk HTML", () => { + assert.equal( + htmlToText("

Bill text

Section 1.

"), + "Bill text Section 1.", + ); +}); + +void test("parsed Texas data satisfies the shared persisted bill contract", () => { + const parsed = parseTexasBillHistory(fixture, "89R"); + assert.equal( + CreateBillSchema.safeParse({ + ...parsed, + sourceWebsite: "capitol.texas.gov", + }).success, + true, + ); +}); diff --git a/apps/scraper/src/scrapers/texas-legislature.ts b/apps/scraper/src/scrapers/texas-legislature.ts new file mode 100644 index 00000000..92a26c58 --- /dev/null +++ b/apps/scraper/src/scrapers/texas-legislature.ts @@ -0,0 +1,164 @@ +import type { BillData, Scraper } from "../utils/types.js"; +import type { TexasBulkClient } from "./texas-legislature-source.js"; +import { setExpectedTotal } from "../utils/db/metrics.js"; +import { upsertContent } from "../utils/db/operations.js"; +import { createLogger } from "../utils/log.js"; +import { + htmlToText, + openStatesSessionName, + parseTexasBillHistory, + TEXAS_JURISDICTION, +} from "./texas-legislature-parser.js"; +import { + bulkHtmlPath, + listFilesRecursively, + selectCurrentTexasSession, + TexasFtpClient, +} from "./texas-legislature-source.js"; +import { texasLegislatureConfig } from "./texas-legislature.config.js"; + +const logger = createLogger("texas-legislature"); + +interface OpenStatesSearchResponse { + results?: { id: string; identifier: string; session: string }[]; +} + +export async function matchOpenStatesBillId( + session: string, + billNumber: string, + apiKey = process.env.OPEN_STATES_API_KEY, +): Promise { + if (!apiKey) return undefined; + const url = new URL("https://v3.openstates.org/bills"); + url.searchParams.set("jurisdiction", TEXAS_JURISDICTION); + url.searchParams.set("session", openStatesSessionName(session)); + url.searchParams.set("q", billNumber); + url.searchParams.set("per_page", "5"); + const response = await fetch(url, { + headers: { Accept: "application/json", "X-API-KEY": apiKey }, + }); + if (!response.ok) return undefined; + const payload = (await response.json()) as OpenStatesSearchResponse; + return payload.results?.find( + (bill) => + bill.identifier.replace(/\s+/g, "").toUpperCase() === + billNumber.replace(/\s+/g, "").toUpperCase() && + bill.session === openStatesSessionName(session), + )?.id; +} + +async function enrichDocuments( + client: TexasBulkClient, + session: string, + documents: ReturnType["documents"], +) { + return Promise.all( + documents.map(async (document) => { + const path = bulkHtmlPath(session, document); + if (!path) return document; + try { + const html = (await client.download(path)).toString("utf8"); + const text = htmlToText(html); + return text ? { ...document, text } : document; + } catch (error) { + logger.debug( + `Optional bulk HTML unavailable at ${path}: ${error instanceof Error ? error.message : error}`, + ); + return document; + } + }), + ); +} + +export async function scrapeTexasLegislature( + client: TexasBulkClient, + options: { maxItems: number; session?: string }, +): Promise { + const availableSessions = (await client.list("/bills")) + .filter((entry) => entry.isDirectory) + .map((entry) => entry.name); + const currentSession = selectCurrentTexasSession(availableSessions); + const session = options.session?.toUpperCase() ?? currentSession; + if (session !== currentSession) { + throw new Error( + `Texas ingestion is current-session only: requested ${session}, latest bulk session is ${currentSession}`, + ); + } + + logger.info(`Reading official Texas bulk data for ${session}...`); + const paths = ( + await listFilesRecursively(client, `/bills/${session}/billhistory`) + ) + .filter( + (path) => + path.toLowerCase().endsWith(".xml") && + !/\/history(?:_periodic)?\.xml$/i.test(path), + ) + .sort() + .slice(0, options.maxItems); + setExpectedTotal(paths.length); + let persisted = 0; + + for (const path of paths) { + try { + const parsed = parseTexasBillHistory( + (await client.download(path)).toString("utf8"), + session, + ); + const documents = await enrichDocuments( + client, + session, + parsed.documents, + ); + const latestBillText = documents + .filter((document) => document.type === "bill_text" && document.text) + .at(-1)?.text; + let openStatesId: string | undefined; + try { + openStatesId = await matchOpenStatesBillId(session, parsed.billNumber); + } catch (error) { + logger.debug( + `Open States identity match skipped for ${parsed.billNumber}: ${error instanceof Error ? error.message : error}`, + ); + } + const data: BillData = { + ...parsed, + documents, + ...(latestBillText && { fullText: latestBillText }), + ...(openStatesId && { openStatesId }), + sourceWebsite: "capitol.texas.gov", + }; + await upsertContent({ type: "bill", data }, { skipEnrichment: true }); + persisted += 1; + } catch (error) { + logger.error( + `Failed to process ${path}: ${error instanceof Error ? error.message : error}`, + ); + } + } + logger.success( + `Persisted ${persisted}/${paths.length} Texas bills from ${session}.`, + ); +} + +async function scrape(options?: { maxItems?: number }): Promise { + const client = new TexasFtpClient(); + try { + await client.connect(); + await scrapeTexasLegislature(client, { + maxItems: + options?.maxItems ?? + (Number(process.env.TEXAS_LEGISLATURE_MAX_ITEMS) || 100), + ...(process.env.TEXAS_LEGISLATURE_SESSION && { + session: process.env.TEXAS_LEGISLATURE_SESSION, + }), + }); + } finally { + client.close(); + } +} + +export const texasLegislature: Scraper = { + ...texasLegislatureConfig, + scrape, +}; diff --git a/apps/scraper/src/utils/db/helpers.ts b/apps/scraper/src/utils/db/helpers.ts index 0d0ce7a7..2e1aaa83 100644 --- a/apps/scraper/src/utils/db/helpers.ts +++ b/apps/scraper/src/utils/db/helpers.ts @@ -20,6 +20,7 @@ const logger = createLogger("db"); export async function checkExistingBill( billNumber: string, sourceWebsite: string, + legislativeSession = "", ): Promise { try { const [existing] = await db @@ -30,7 +31,13 @@ export async function checkExistingBill( thumbnailUrl: Bill.thumbnailUrl, }) .from(Bill) - .where(and(eq(Bill.billNumber, billNumber), eq(Bill.sourceWebsite, sourceWebsite))) + .where( + and( + eq(Bill.billNumber, billNumber), + eq(Bill.sourceWebsite, sourceWebsite), + eq(Bill.legislativeSession, legislativeSession), + ), + ) .limit(1); if (!existing) { diff --git a/apps/scraper/src/utils/db/operations.ts b/apps/scraper/src/utils/db/operations.ts index 52a4387d..3ab5d39f 100644 --- a/apps/scraper/src/utils/db/operations.ts +++ b/apps/scraper/src/utils/db/operations.ts @@ -70,6 +70,14 @@ function hashFields(input: ContentData): string { status: input.data.status, summary: input.data.summary, fullText: input.data.fullText, + jurisdiction: input.data.jurisdiction, + legislativeSession: input.data.legislativeSession, + openStatesId: input.data.openStatesId, + subjects: input.data.subjects, + sponsorships: input.data.sponsorships, + documents: input.data.documents, + votes: input.data.votes, + actions: input.data.actions, }); case "government_content": return JSON.stringify({ @@ -90,7 +98,11 @@ function hashFields(input: ContentData): string { async function checkExisting(input: ContentData) { switch (input.type) { case "bill": - return checkExistingBill(input.data.billNumber, input.data.sourceWebsite); + return checkExistingBill( + input.data.billNumber, + input.data.sourceWebsite, + input.data.legislativeSession, + ); case "government_content": return checkExistingGovernmentContent(input.data.url); case "court_case": @@ -111,7 +123,7 @@ function getUpdateTable(input: ContentData) { export async function upsertContent( input: ContentData, - options?: { newItemLimiter?: NewItemLimiter }, + options?: { newItemLimiter?: NewItemLimiter; skipEnrichment?: boolean }, ) { const newContentHash = createContentHash(hashFields(input)); const existing = await checkExisting(input); @@ -232,7 +244,11 @@ export async function upsertContent( versions: [], }) .onConflictDoUpdate({ - target: [Bill.billNumber, Bill.sourceWebsite], + target: [ + Bill.billNumber, + Bill.sourceWebsite, + Bill.legislativeSession, + ], set: { title: d.title, description: d.description, @@ -241,8 +257,16 @@ export async function upsertContent( introducedDate: d.introducedDate, congress: d.congress, chamber: d.chamber, + jurisdiction: d.jurisdiction, + legislativeSession: d.legislativeSession, + openStatesId: d.openStatesId, + subjects: d.subjects, + sponsorships: d.sponsorships, + documents: d.documents, + votes: d.votes, summary: d.summary, fullText: d.fullText, + actions: d.actions, url: d.url, contentHash: newContentHash, updatedAt: new Date(), @@ -312,6 +336,15 @@ export async function upsertContent( return result; } + if (options?.skipEnrichment) { + tickProgress({ + newEntries: progressKind === "new" ? 1 : 0, + unchanged: progressKind === "unchanged" ? 1 : 0, + changed: progressKind === "changed" ? 1 : 0, + }); + return result; + } + // Phase 2: AI enrichment — skipped entirely if rate-limited try { const existingDescription = sourceDescription || persistedDescription; diff --git a/docs/README.md b/docs/README.md index af74cdcc..e59da397 100644 --- a/docs/README.md +++ b/docs/README.md @@ -4,15 +4,16 @@ Start with [CONTRIBUTING.md](../CONTRIBUTING.md) for dev setup. These docs go de ## How the system works -| Doc | What it covers | -| ---------------------------------------------------- | ---------------------------------------------------------------------------------------------------- | -| [Architecture overview](./architecture.md) | The big picture: system diagram, monorepo layout, build tooling, and why the key decisions were made | -| [Data layer](./data-layer.md) | Drizzle + Supabase Postgres, the schema (~20 tables), migrations | -| [API layer](./api.md) | The tRPC router, civic data integrations, request path, LLM provider | -| [Ballot-measure enrichment](./measure-enrichment.md) | How measure summaries are cross-validated across official sources, adapter by adapter | -| [Candidate enrichment](./candidate-enrichment.md) | The same cross-validation pattern applied to candidate bios/photos/contact info | -| [Scraper pipeline](./scraper.md) | The standalone content scraper: sources, change detection, AI generation | -| [Frontend apps](./frontend.md) | Expo mobile app, Next.js web, shared UI, cross-platform auth | +| Doc | What it covers | +| ---------------------------------------------------------- | ---------------------------------------------------------------------------------------------------- | +| [Architecture overview](./architecture.md) | The big picture: system diagram, monorepo layout, build tooling, and why the key decisions were made | +| [Data layer](./data-layer.md) | Drizzle + Supabase Postgres, the schema (~20 tables), migrations | +| [API layer](./api.md) | The tRPC router, civic data integrations, request path, LLM provider | +| [Ballot-measure enrichment](./measure-enrichment.md) | How measure summaries are cross-validated across official sources, adapter by adapter | +| [Candidate enrichment](./candidate-enrichment.md) | The same cross-validation pattern applied to candidate bios/photos/contact info | +| [Texas current-election data](./texas-current-election.md) | Current-cycle SOS results/candidates and separately cited TLC amendment analyses | +| [Scraper pipeline](./scraper.md) | The standalone content scraper: sources, change detection, AI generation | +| [Frontend apps](./frontend.md) | Expo mobile app, Next.js web, shared UI, cross-platform auth | ## How to do things diff --git a/docs/api.md b/docs/api.md index 2f6381a2..0025ebb1 100644 --- a/docs/api.md +++ b/docs/api.md @@ -13,17 +13,17 @@ The API is a [tRPC v11](https://trpc.io/) router in `packages/api/`, served by N The root router (`packages/api/src/root.ts`) composes **nine** sub-routers: -| Router | Procedures (Q = query, M = mutation, 🔒 = protected) | -| ------------ | ---------------------------------------------------------------------------------------------------------------------------------------------- | -| `auth` | `getSession` (Q), `getSecretMessage` (Q 🔒) | -| `civic` | `getElections`, `getVoterInfo`, `getRepresentatives`, `getRepresentativesEnriched` (all Q) — Google Civic + measure/candidate cross-validation | -| `places` | `autocomplete` (Q), `details` (M) — Google Places address autocomplete for the ballot lookup | -| `legistar` | `getLocalBills`, `getMeetings`, `getAgenda`, `getVotes`, `getBodies`, `getMeetingVotes` (all Q) — local councils | -| `openStates` | `searchBills`, `getBillDetails`, `getLegislators`, `getBillVotes` (all Q) — CA state legislature (Open States v3) | -| `content` | `getAll`, `getByType`, `getById` (all Q) — aggregates bill / government_content / court_case | -| `video` | `getInfinite` (Q) — cursor-paginated feed; converts `bytea` images to data URIs | -| `post` | `all`, `byId` (Q); `create`, `delete` (M 🔒) | -| `user` | preferences, blocked content, settings, profile, and saved-article CRUD (all 🔒) | +| Router | Procedures (Q = query, M = mutation, 🔒 = protected) | +| ------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `auth` | `getSession` (Q), `getSecretMessage` (Q 🔒) | +| `civic` | `getElections`, `getVoterInfo`, `getTexasCurrentElection`, `getRepresentatives`, `getRepresentativesEnriched` (all Q) — Google Civic + official current-cycle Texas data + enrichment | +| `places` | `autocomplete` (Q), `details` (M) — Google Places address autocomplete for the ballot lookup | +| `legistar` | `getLocalBills`, `getMeetings`, `getAgenda`, `getVotes`, `getBodies`, `getMeetingVotes` (all Q) — local councils | +| `openStates` | `searchBills`, `getBillDetails`, `getLegislators`, `getBillVotes` (all Q) — CA state legislature (Open States v3) | +| `content` | `getAll`, `getByType`, `getById` (all Q) — aggregates bill / government_content / court_case | +| `video` | `getInfinite` (Q) — cursor-paginated feed; converts `bytea` images to data URIs | +| `post` | `all`, `byId` (Q); `create`, `delete` (M 🔒) | +| `user` | preferences, blocked content, settings, profile, and saved-article CRUD (all 🔒) | ## Civic Data & External Sources @@ -32,11 +32,19 @@ The `civic` router calls the **Google Civic Information API** (`GOOGLE_CIVIC_API Other live civic integrations: - **`legistar`** — scrapes Legistar instances for San Jose, Santa Clara County, and Sunnyvale (no key required). +- **`localGovernment`** — reads persisted provider-neutral meetings. `listMeetings` returns bounded meeting summaries and official links; `getMeeting` returns agenda items, actions, attachments, and named votes without live source calls. - **`openStates`** — California bills, legislators, and votes via the Open States v3 API (`OPEN_STATES_API_KEY`). - **`places`** — Google **Places Autocomplete (New)** for the ballot address entry (`packages/api/src/lib/places.ts`). `autocomplete` returns US street-address predictions (biased `includedRegionCodes: ["us"]`, `includedPrimaryTypes: street_address/premise/subpremise`) for queries ≥3 chars; `details` resolves a `placeId` to its full `formattedAddress` (the ZIP the prediction omits, which Civic wants). A **session token** (UUID stable across one address entry) bundles all keystroke calls plus the closing `details` into a single billed unit. Reuses `GOOGLE_PLACES_API_KEY` → `GOOGLE_API_KEY` → `GOOGLE_CIVIC_API_KEY`; with no key it serves a small mock list so the dropdown still works in dev (same fallback pattern as `civic`). Ballot measures and candidates returned by `getVoterInfo` are run through cross-validation engines that merge multiple public-record sources by trust tier — see [Ballot-measure enrichment](./measure-enrichment.md) and [Candidate enrichment](./candidate-enrichment.md). Key/access setup for every source is in [Civic data source setup](./civic-data-sources.md). +`civic.getTexasCurrentElection` is an input-free reader over persisted current +Texas SOS/TLC snapshots. It returns current statewide/federal/district +candidates and results plus the latest constitutional-amendment analyses. SOS +facts and TLC explanation are separately cited. Historical election browsing is +intentionally not part of this procedure; see +[Texas current-election data](./texas-current-election.md). + ## LLM Provider `packages/api/src/lib/ai-provider.ts` exports a single swappable `llm` via the Vercel AI SDK: **Groq** when `GROQ_API_KEY` is set, then **OpenRouter** when `OPENROUTER_API_KEY` is present (using `OPENROUTER_MODEL`, default `deepseek/deepseek-v4-flash`), then **OpenAI `gpt-4o-mini`**, then deprecated direct **DeepSeek `deepseek-v4-flash`** during migration, and `null` otherwise. Callers treat `null` as "AI unavailable" and skip generation rather than throw. diff --git a/docs/civic-data-sources.md b/docs/civic-data-sources.md index ba126200..82c9400a 100644 --- a/docs/civic-data-sources.md +++ b/docs/civic-data-sources.md @@ -9,6 +9,7 @@ How to obtain keys/access for every civic integration. For local dev, copy `.env | Google Places (address autocomplete) | Yes | Pay-as-you-go | `GOOGLE_PLACES_API_KEY` (→ `GOOGLE_API_KEY` → `GOOGLE_CIVIC_API_KEY`) | | Vote Smart | Yes | Free (org tier) | `VOTE_SMART_API_KEY` | | Legistar (local councils) | No | Free | — | +| Cedar Park council records | No | Free | — | | VOTE411 / LWV (scraper) | No | Free | — | | CA SOS Voter Guide (scraper) | No | Free | — | | Santa Clara measure pipeline (scraper) | No | Free | — | @@ -27,6 +28,15 @@ How to obtain keys/access for every civic integration. For local dev, copy `.env **Legistar Web API** — local council meetings, legislation, votes, agendas; no key, unlimited public data. Supported jurisdictions (client id): San Jose (`sanjose`), Santa Clara County (`santaclara`), Sunnyvale (`sunnyvale`). Add a city by extracting its `*.legistar.com` subdomain into the `JURISDICTIONS` constant in `packages/api/src/integrations/legistar.ts`. Base: (OData-compatible). +**Cedar Park City Council** — the official CivicEngage City Council page embeds +a Municode Meetings publish page. The `cedar-park-council` scraper follows that +embed for the latest 12 months, uses deterministic HTML/PDF parsing, and writes +the provider-neutral local-government tables read by the `localGovernment` +tRPC router. No AI key is needed. Configuration lives in +`apps/scraper/src/scrapers/civicengage.config.ts`; a second jurisdiction using +the same embed supplies a new CivicEngage host/path, timezone, Municode +`cid`/`ppid`, and body matcher. + **VOTE411 / League of Women Voters** — nonpartisan voter guides, candidate questionnaires, measure explanations; no key (scraper, rate-limited + cached). ⚠️ ToS bars commercial use without **written consent** and prohibits automated queries — a negotiated partnership is the only compliant path (tracked in the Outreach Tracker). Also `cavotes.org/easy-voter-guide/` for CA. **CA SOS Official Voter Guide (scraper)** — official statewide proposition summaries (AG), pro/con arguments, full-text links; no key. Source `voterguide.sos.ca.gov`. Distinct from the (dead) results API — this carries the human-written official measure content and feeds the cross-validation engine (`measure-sources/ca-sos-voterguide.ts`). CA statewide only; local measures use the county pipelines. diff --git a/docs/data-layer.md b/docs/data-layer.md index 76bf2357..0190b2f0 100644 --- a/docs/data-layer.md +++ b/docs/data-layer.md @@ -67,16 +67,25 @@ All three content tables share a common pattern: **Civic / elections** — the voter-information model: -| Table | Purpose | -| ------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `election` | Election records (external id, date, type, OCD division, deadlines JSONB) | -| `contest` | Races _and_ ballot measures. `type` = candidate \| referendum. For measures: `referendum_title`, pro/con statements, `summary`, `summary_is_ai_generated`, `fiscal_impact`, and a `citations` JSONB array (per-field source attribution: field, source name/url, trust tier, official flag) | -| `candidate` | Candidates within a contest (party, incumbent, contact, bio) | -| `polling_location` | Polling places / early-vote sites / drop boxes, geo-located (lat/long), with hours | -| `role_description` | Reusable descriptions of offices/roles by level (seeded with ~18 federal→local roles) | +| Table | Purpose | +| -------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `election` | Election records (external id, date, type, OCD division, deadlines JSONB) | +| `contest` | Races _and_ ballot measures. `type` = candidate \| referendum. For measures: `referendum_title`, pro/con statements, `summary`, `summary_is_ai_generated`, `fiscal_impact`, and a `citations` JSONB array (per-field source attribution: field, source name/url, trust tier, official flag) | +| `candidate` | Candidates within a contest (party, incumbent, contact, bio) | +| `polling_location` | Polling places / early-vote sites / drop boxes, geo-located (lat/long), with hours | +| `role_description` | Reusable descriptions of offices/roles by level (seeded with ~18 federal→local roles) | +| `election_source_snapshot` | Provider-neutral, idempotent current-cycle election snapshots with source version, checksum, diagnostics, and source URLs. Texas SOS facts and TLC analyses occupy separate rows. | **Local government (Legistar cache)** — `legistar_body`, `legistar_matter`, `legistar_meeting`, `legistar_agenda_item`, `legistar_vote`. These cache San Jose / Santa Clara / Sunnyvale council data (ordinances, meetings, agenda items, votes) keyed by `(jurisdiction, *_id)` with a `fetched_at` timestamp. +**Provider-neutral local government** — `local_government_meeting`, +`local_government_document`, `local_government_agenda_item`, and +`local_government_vote`. These hold systems that do not expose Legistar +semantics, including Durham's OnBase instance and Cedar Park's CivicEngage +source. Meetings are keyed by `(source, jurisdiction, external_id)`; documents +are versionable rows and agenda items carry official motion, outcome, and vote +text. The API reads persisted rows instead of scraping during a user request. + **User engagement & caching:** | Table | Purpose | diff --git a/docs/data-sources-api.md b/docs/data-sources-api.md index fd61ad4e..2bb929b7 100644 --- a/docs/data-sources-api.md +++ b/docs/data-sources-api.md @@ -8,16 +8,16 @@ All live-API functions come from `@acme/api`. Scrapers run as CLI jobs in `apps/ ## Quick index -| Category | What you get | Import | -|---|---|---| -| [Ballot & elections](#ballot--elections) | Contests, candidates, polling places, ballot measures | `@acme/api` | -| [Ballot-measure enrichment](#ballot-measure-enrichment) | Summaries, fiscal impact, pro/con arguments, citations | `@acme/api` | -| [State legislation](#state-legislation) | CA bills, legislators, votes | `@acme/api` | -| [Local government](#local-government) | City/county meetings, ordinances, votes | `@acme/api` | -| [CA election results](#ca-election-results) | Statewide vote counts (live JSON feed) | `@acme/api` | -| [Address autocomplete](#address-autocomplete) | Street-address suggestions + place details | `@acme/api` | -| [Corpus content (DB)](#corpus-content-db) | Bills, court cases, government content articles | tRPC `content` router | -| [Scrapers (background jobs)](#scrapers-background-jobs) | Congress bills, SCOTUS, Federal Register, LAO, CA SOS | `apps/scraper` CLI | +| Category | What you get | Import | +| ------------------------------------------------------- | ------------------------------------------------------ | --------------------- | +| [Ballot & elections](#ballot--elections) | Contests, candidates, polling places, ballot measures | `@acme/api` | +| [Ballot-measure enrichment](#ballot-measure-enrichment) | Summaries, fiscal impact, pro/con arguments, citations | `@acme/api` | +| [State legislation](#state-legislation) | CA bills, legislators, votes | `@acme/api` | +| [Local government](#local-government) | City/county meetings, ordinances, votes | `@acme/api` | +| [CA election results](#ca-election-results) | Statewide vote counts (live JSON feed) | `@acme/api` | +| [Address autocomplete](#address-autocomplete) | Street-address suggestions + place details | `@acme/api` | +| [Corpus content (DB)](#corpus-content-db) | Bills, court cases, government content articles | tRPC `content` router | +| [Scrapers (background jobs)](#scrapers-background-jobs) | Congress bills, SCOTUS, Federal Register, LAO, CA SOS | `apps/scraper` CLI | --- @@ -32,26 +32,27 @@ All live-API functions come from `@acme/api`. Scrapers run as CLI jobs in `apps/ ```ts import { + getDistrictElectionResults, + getElectionResults, getElections, getVoterInfo, - getElectionResults, - getDistrictElectionResults, } from "@acme/api"; // Or, for direct tRPC consumption: // civic.getVoterInfo({ address, electionId }) ``` -| Function | Args | Returns | -|---|---|---| -| `getElections()` | — | `Election[]` — all elections Google knows about | -| `getVoterInfo(address, electionId)` | street address string, election ID | `VoterInfoResponse` — contests, polling places, drop boxes, enriched measures | -| `getElectionResults(stateFips, countyFips)` | FIPS codes | CA SOS results for the matching jurisdiction | -| `getDistrictElectionResults(districtRef)` | `DistrictRef` | narrowed by district | +| Function | Args | Returns | +| ------------------------------------------- | ---------------------------------- | ----------------------------------------------------------------------------- | +| `getElections()` | — | `Election[]` — all elections Google knows about | +| `getVoterInfo(address, electionId)` | street address string, election ID | `VoterInfoResponse` — contests, polling places, drop boxes, enriched measures | +| `getElectionResults(stateFips, countyFips)` | FIPS codes | CA SOS results for the matching jurisdiction | +| `getDistrictElectionResults(districtRef)` | `DistrictRef` | narrowed by district | **`getVoterInfo` runs the full enrichment pipeline** — ballot measures come back with summaries, fiscal impact, pro/con arguments, and per-field citations from the cross-validation engine. The raw Google Civic response alone omits nearly all content fields; this is what fills them. Key return types (all exported from `@acme/api`): + - `Contest` — a race or measure on the ballot (includes enriched `CanonicalMeasure` fields when present) - `Candidate` — name, party, photo, channels, incumbency, Vote Smart bio - `PollingLocation` — address, hours, type @@ -67,22 +68,21 @@ Key return types (all exported from `@acme/api`): **When to call directly:** when you need enriched measure data outside the `getVoterInfo` flow — e.g. article generation, batch enrichment scripts ```ts -import { - crossValidateMeasure, - type CanonicalMeasure, - type MeasureSourceData, - type MeasureCitation, - type SourceTier, - SOURCE_TIER_RANK, +import type { + CanonicalMeasure, + MeasureCitation, + MeasureSourceData, + SourceTier, } from "@acme/api"; +import type { CanonicalMeasure } from "@acme/api/lib/measure-sources/types"; +import { crossValidateMeasure, SOURCE_TIER_RANK } from "@acme/api"; // or via the named subpath: import { crossValidateMeasure } from "@acme/api/lib/measure-crossvalidate"; -import type { CanonicalMeasure } from "@acme/api/lib/measure-sources/types"; const result: CanonicalMeasure = await crossValidateMeasure( { title: "Proposition 1", - subtitle: "...", // optional — Google Civic's own text, used as last-resort + subtitle: "...", // optional — Google Civic's own text, used as last-resort text: "...", url: "...", }, @@ -96,32 +96,32 @@ const result: CanonicalMeasure = await crossValidateMeasure( `CanonicalMeasure` fields: -| Field | Type | Notes | -|---|---|---| -| `title` | `string` | Measure title (passed through) | -| `summary` | `string \| undefined` | Best available summary; cite `citations` for the source | -| `summaryShort` | `string \| undefined` | One-sentence card preview | -| `summaryLong` | `string \| undefined` | Fuller detail-screen paragraph | -| `summaryIsAiGenerated` | `boolean` | True when AI grounded on fetched text — UI should label it | -| `fiscalImpact` | `string \| undefined` | Official fiscal analysis (LAO for CA props, else Ballotpedia) | -| `fullText` | `string \| undefined` | Full measure text when available | -| `fullTextUrl` | `string \| undefined` | Link to the official text | -| `proArguments` | `MeasureArgument[]` | Attributed pro arguments | -| `conArguments` | `MeasureArgument[]` | Attributed con arguments | -| `citations` | `MeasureCitation[]` | One entry per populated field — `{field, sourceName, sourceUrl, tier, official}` | -| `discrepancies` | `string[] \| undefined` | Fields where top-2 sources disagreed; for human review | +| Field | Type | Notes | +| ---------------------- | ----------------------- | -------------------------------------------------------------------------------- | +| `title` | `string` | Measure title (passed through) | +| `summary` | `string \| undefined` | Best available summary; cite `citations` for the source | +| `summaryShort` | `string \| undefined` | One-sentence card preview | +| `summaryLong` | `string \| undefined` | Fuller detail-screen paragraph | +| `summaryIsAiGenerated` | `boolean` | True when AI grounded on fetched text — UI should label it | +| `fiscalImpact` | `string \| undefined` | Official fiscal analysis (LAO for CA props, else Ballotpedia) | +| `fullText` | `string \| undefined` | Full measure text when available | +| `fullTextUrl` | `string \| undefined` | Link to the official text | +| `proArguments` | `MeasureArgument[]` | Attributed pro arguments | +| `conArguments` | `MeasureArgument[]` | Attributed con arguments | +| `citations` | `MeasureCitation[]` | One entry per populated field — `{field, sourceName, sourceUrl, tier, official}` | +| `discrepancies` | `string[] \| undefined` | Fields where top-2 sources disagreed; for human review | **Sources wired into the engine** (in trust-tier order): -| Source | Tier | Scope | -|---|---|---| -| CA SOS Official Voter Guide | `state_sos` | CA statewide props — AG summary, official pro/con | -| CA LAO Fiscal Analyses | `state_sos` | CA statewide props — nonpartisan fiscal impact | -| LWV / CaVotes | `lwv` | CA statewide props — nonpartisan pros & cons | -| Ballotpedia | `ballotpedia` | Statewide + local lettered measures (all states) | -| Wikipedia | `wikipedia` | CA statewide props — encyclopedic overview | -| Vote Smart | `vote_smart` | State-level measures — summary + pro/con URLs | -| Google Civic | `google_civic` | The original input — lowest trust, always present | +| Source | Tier | Scope | +| --------------------------- | -------------- | ------------------------------------------------- | +| CA SOS Official Voter Guide | `state_sos` | CA statewide props — AG summary, official pro/con | +| CA LAO Fiscal Analyses | `state_sos` | CA statewide props — nonpartisan fiscal impact | +| LWV / CaVotes | `lwv` | CA statewide props — nonpartisan pros & cons | +| Ballotpedia | `ballotpedia` | Statewide + local lettered measures (all states) | +| Wikipedia | `wikipedia` | CA statewide props — encyclopedic overview | +| Vote Smart | `vote_smart` | State-level measures — summary + pro/con URLs | +| Google Civic | `google_civic` | The original input — lowest trust, always present | When no human source has a summary, the engine falls back to grounded AI (SPUR Bay Area voter guide as source text), flagged `summaryIsAiGenerated: true`. @@ -135,55 +135,80 @@ When no human source has a summary, the engine falls back to grounded AI (SPUR B **Scope:** California state bills and legislators (expandable to other states) ```ts +import type { + GetBillsOptions, + OpenStatesBill, + OpenStatesPerson, +} from "@acme/api"; import { - getBills, getBillDetails, - getLegislators, - getVotes, + getBills, + getBillsBySponsor, getCurrentSessions, getLegislatorById, - getBillsBySponsor, + getLegislators, + getVotes, openStatesClient, - type OpenStatesBill, - type OpenStatesPerson, - type GetBillsOptions, } from "@acme/api"; // or import { getBills } from "@acme/api/clients/open-states"; -const bills = await getBills({ state: "ca", session: "20232024", query: "housing" }); +const bills = await getBills({ + state: "ca", + session: "20232024", + query: "housing", +}); const detail = await getBillDetails("ocd-bill/..."); const legislators = await getLegislators({ state: "ca" }); ``` -| Function | What it returns | -|---|---| -| `getBills(opts)` | `OpenStatesBillSearchResult[]` — paginated bill search | -| `getBillDetails(billId)` | `OpenStatesBill` — full bill with actions, sponsors, versions | -| `getLegislators(opts)` | `OpenStatesPerson[]` — legislators matching the query | -| `getVotes(billId)` | `OpenStatesVote[]` — roll-call votes for a bill | -| `getCurrentSessions(state)` | Active legislative sessions | -| `getBillsBySponsor(personId)` | Bills sponsored by a legislator | -| `openStatesClient` | Raw client for custom queries | +| Function | What it returns | +| ----------------------------- | ------------------------------------------------------------- | +| `getBills(opts)` | `OpenStatesBillSearchResult[]` — paginated bill search | +| `getBillDetails(billId)` | `OpenStatesBill` — full bill with actions, sponsors, versions | +| `getLegislators(opts)` | `OpenStatesPerson[]` — legislators matching the query | +| `getVotes(billId)` | `OpenStatesVote[]` — roll-call votes for a bill | +| `getCurrentSessions(state)` | Active legislative sessions | +| `getBillsBySponsor(personId)` | Bills sponsored by a legislator | +| `openStatesClient` | Raw client for custom queries | --- ## Local government -**Source:** Legistar Web API -**Entry point:** `packages/api/src/integrations/legistar.ts` +**Sources:** persisted local-government records plus the Legistar Web API +**Entry points:** `packages/api/src/lib/local-government.ts`, `packages/api/src/integrations/legistar.ts` **Auth:** None (public API) -**Jurisdictions wired:** San Jose (`sanjose`), Santa Clara County (`santaclara`), Sunnyvale (`sunnyvale`) +**Persisted jurisdiction:** Cedar Park (`cedar-park-tx`); live Legistar jurisdictions: San Jose, Santa Clara County, Sunnyvale + +The product-facing reader uses normalized persisted records. Cedar Park's +scheduled scraper populates these from the city's official CivicEngage page and +embedded Municode Meetings publication: ```ts import { - legistar, - LegistarClient, - JURISDICTIONS, - type LegistarMeeting, - type LegistarMatter, - type LegistarVote, + getLocalGovernmentMeeting, + getLocalGovernmentMeetings, } from "@acme/api"; + +const meetings = await getLocalGovernmentMeetings({ + jurisdiction: "cedar-park-tx", + limit: 20, +}); +const detail = meetings[0] + ? await getLocalGovernmentMeeting(meetings[0].id) + : null; +``` + +The equivalent tRPC procedures are `localGovernment.meetings` and +`localGovernment.meeting`. Detail includes official/versioned documents, +agenda items, motions, outcomes, tally text, and named votes when published. + +Legistar remains available as a provider-specific live client: + +```ts +import type { LegistarMatter, LegistarMeeting, LegistarVote } from "@acme/api"; +import { JURISDICTIONS, legistar, LegistarClient } from "@acme/api"; // or import { legistar } from "@acme/api/integrations/legistar"; @@ -192,16 +217,34 @@ const matters = await legistar.getMatters("santaclara", { keywords: "budget" }); const votes = await legistar.getVotes("sanjose", matterId); ``` -| Method | Returns | -|---|---| -| `legistar.getMeetings(jurisdiction, opts?)` | `LegistarMeeting[]` — council meetings with agendas | -| `legistar.getMatters(jurisdiction, opts?)` | `LegistarMatter[]` — ordinances, resolutions, items | -| `legistar.getVotes(jurisdiction, matterId)` | `LegistarVote[]` — how each member voted | -| `legistar.getBodies(jurisdiction)` | `LegistarBody[]` — committees and boards | -| `legistar.getAttachments(jurisdiction, matterId)` | `LegistarAttachment[]` — PDFs, staff reports | +| Method | Returns | +| ------------------------------------------------- | --------------------------------------------------- | +| `legistar.getMeetings(jurisdiction, opts?)` | `LegistarMeeting[]` — council meetings with agendas | +| `legistar.getMatters(jurisdiction, opts?)` | `LegistarMatter[]` — ordinances, resolutions, items | +| `legistar.getVotes(jurisdiction, matterId)` | `LegistarVote[]` — how each member voted | +| `legistar.getBodies(jurisdiction)` | `LegistarBody[]` — committees and boards | +| `legistar.getAttachments(jurisdiction, matterId)` | `LegistarAttachment[]` — PDFs, staff reports | To add a city: add its `*.legistar.com` subdomain to `JURISDICTIONS` in `integrations/legistar.ts`. +### Provider-neutral local meetings + +- **Source:** [City of Durham OnBase Agenda Online](https://cityordinances.durhamnc.gov/OnBaseAgendaOnline/) +- **Reader:** `packages/api/src/lib/local-government.ts` +- **tRPC:** `localGovernment.meetings`, `localGovernment.meeting` + +The Durham and Cedar Park scrapers persist meetings, agenda or minutes item +outlines, supporting-document URLs, and official action/vote text. The reader +is provider-neutral and serves only cached database records; it does not +contact upstream meeting systems in the request path. + +```ts +const meetings = await getLocalGovernmentMeetings({ + jurisdiction: "durham-nc", +}); +const meeting = await getLocalGovernmentMeeting(meetings[0].id); +``` + --- ## CA election results @@ -211,11 +254,8 @@ To add a city: add its `*.legistar.com` subdomain to `JURISDICTIONS` in `integra **Auth:** None ```ts -import { - SOS_RESULTS_HOME, - type ElectionContestResult, - type ResultCandidate, -} from "@acme/api"; +import type { ElectionContestResult, ResultCandidate } from "@acme/api"; +import { SOS_RESULTS_HOME } from "@acme/api"; // or import { SOS_RESULTS_HOME } from "@acme/api/clients/ca-sos-results"; ``` @@ -231,11 +271,8 @@ import { SOS_RESULTS_HOME } from "@acme/api/clients/ca-sos-results"; **Auth:** `GOOGLE_PLACES_API_KEY` (falls back through `GOOGLE_API_KEY` → `GOOGLE_CIVIC_API_KEY`) ```ts -import { - getAddressSuggestions, - getPlaceDetails, - type AddressSuggestion, -} from "@acme/api"; +import type { AddressSuggestion } from "@acme/api"; +import { getAddressSuggestions, getPlaceDetails } from "@acme/api"; const suggestions = await getAddressSuggestions("123 Main St"); const details = await getPlaceDetails(suggestions[0].placeId); @@ -261,24 +298,24 @@ const typed = await trpc.content.getByType.query({ type: "court_case" }); `ContentCard` shape (returned by all three procedures): -| Field | Type | -|---|---| -| `id` | `string` | -| `title` | `string` | -| `description` | `string \| null` | -| `type` | `"bill" \| "court_case" \| "government_content"` | -| `isAIGenerated` | `boolean` | -| `thumbnailUrl` | `string \| null` | +| Field | Type | +| --------------- | ------------------------------------------------ | +| `id` | `string` | +| `title` | `string` | +| `description` | `string \| null` | +| `type` | `"bill" \| "court_case" \| "government_content"` | +| `isAIGenerated` | `boolean` | +| `thumbnailUrl` | `string \| null` | `getThumbnailForContent(id, type)` is also exported from `@acme/api` for cases where only the thumbnail is needed. **DB tables populated by scrapers:** -| Table | Populated by | Content | -|---|---|---| -| `Bill` | `congress.ts` scraper | Federal bills, actions, sponsor, status, full text | -| `GovernmentContent` | `federalregister.ts` scraper | EOs, proclamations, presidential memos | -| `CourtCase` | `scotus.ts` scraper | SCOTUS opinions via CourtListener | +| Table | Populated by | Content | +| ------------------- | ---------------------------- | -------------------------------------------------- | +| `Bill` | `congress.ts` scraper | Federal bills, actions, sponsor, status, full text | +| `GovernmentContent` | `federalregister.ts` scraper | EOs, proclamations, presidential memos | +| `CourtCase` | `scotus.ts` scraper | SCOTUS opinions via CourtListener | --- @@ -286,15 +323,15 @@ const typed = await trpc.content.getByType.query({ type: "court_case" }); These run as CLI jobs (`bun run scrape -- --scrapers `) in `apps/scraper` and are **not importable as functions at request time**. They populate `CivicApiCache` rows or DB tables that the live API reads. -| Scraper | Populates | Cadence | -|---|---|---| -| `congress` | `Bill` table | Periodic | -| `federalregister` | `GovernmentContent` table | Periodic | -| `scotus` | `CourtCase` table | Periodic | -| `vote411` | `CivicApiCache` (VOTE411 voter guides) | Pre-election | -| `sccCvig` | `CivicApiCache` (SCC county voter guide) | Pre-election | -| `caSosStatements` | `CivicApiCache` (CA SOS candidate statements) | Pre-election | -| `caLaoFiscal` | `CivicApiCache` (LAO fiscal analyses) | Pre-election (~90 days out) | +| Scraper | Populates | Cadence | +| ----------------- | --------------------------------------------- | --------------------------- | +| `congress` | `Bill` table | Periodic | +| `federalregister` | `GovernmentContent` table | Periodic | +| `scotus` | `CourtCase` table | Periodic | +| `vote411` | `CivicApiCache` (VOTE411 voter guides) | Pre-election | +| `sccCvig` | `CivicApiCache` (SCC county voter guide) | Pre-election | +| `caSosStatements` | `CivicApiCache` (CA SOS candidate statements) | Pre-election | +| `caLaoFiscal` | `CivicApiCache` (LAO fiscal analyses) | Pre-election (~90 days out) | **How the cache-warmer pattern works:** a scraper pre-fetches expensive HTML pages and stores structured JSON in `CivicApiCache`. When a user triggers `getVoterInfo`, the measure-source adapters read the cache row (sub-millisecond DB read) instead of doing a live HTML fetch inside the request. Cache TTL is 30 days for LAO; 24 hours for voter info responses. diff --git a/docs/measure-enrichment.md b/docs/measure-enrichment.md index 950a9700..56efd1db 100644 --- a/docs/measure-enrichment.md +++ b/docs/measure-enrichment.md @@ -14,6 +14,7 @@ The Google Civic Information API reliably returns ballot-measure **titles** but ``` CA SOS Official Voter Guide ─┐ +TX SOS facts + TLC analysis ─┤ LWV CaVotes (Pros & Cons) ─┤ Ballotpedia (statewide+local)─┤ Wikipedia (statewide props) ─┼──▶ Cross-Validation Engine ──▶ Canonical Measure ──▶ Cache (CivicApiCache) ──▶ App @@ -31,22 +32,23 @@ The engine fetches all sources concurrently (`Promise.all`) and merges them **fi When more than one source covers the same field, the higher tier wins (defined in `measure-sources/types.ts`, `SOURCE_TIER_RANK`): ``` -county_registrar > state_sos > lwv > ballotpedia > wikipedia > vote_smart > google_civic > ai_generated +county_registrar > state_sos = legislative_council > lwv > ballotpedia > wikipedia > vote_smart > google_civic > ai_generated ``` ## Source adapters (wired today) Each adapter fetches over a shared, defensive helper (`measure-sources/fetch.ts`) that uses a browser User-Agent and turns any failure into `null`. -| Source | Adapter | Tier | Scope / method | -| -------------------------------- | ---------------------- | -------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| CA SOS Official Voter Guide | `ca-sos-voterguide.ts` | `state_sos` | CA statewide props. Official AG summary, pro/con, full-text URL — **real official text, not AI.** Matches on the prop number parsed from the title. The guide is rebuilt each cycle and only serves the active election, so it yields nothing between prop cycles. | -| CA LAO Fiscal Analyses | `ca-lao-fiscal.ts` | `state_sos` | CA statewide props. Official nonpartisan fiscal impact analysis for each proposition. HTML scrape of `lao.ca.gov/BallotAnalysis/Proposition?number=N&year=YYYY` — no API exists. Pre-warmed by the `ca-lao-fiscal` scraper into `CivicApiCache`; adapter reads cache first, falls back to live fetch. Only fires for parsed proposition numbers (not local lettered measures). | -| League of Women Voters — CaVotes | `cavotes.ts` | `lwv` | CA statewide props. Nonpartisan "Pros & Cons" summary, fiscal effects, supporter/opponent arguments via the CaVotes WordPress REST API (`cavotes.org/wp-json/wp/v2/ballots`). Slugs are inconsistent across years, so it enumerates the list and matches by prop number + year. | -| Ballotpedia | `ballotpedia.ts` | `ballotpedia` | **Statewide _and_ local** lettered measures — the main source for local. Rendered article HTML (MediaWiki API disabled), resolved from a year/county index. Extracts ballot summary/question, fiscal impact / impartial analysis, arguments. Year-gated so a same-letter measure from another cycle isn't surfaced. | -| Wikipedia | `wikipedia.ts` | `wikipedia` | CA statewide props only — gated on a parsed prop number (local titles like "Measure Q" collide with unrelated articles). MediaWiki extract for ` California Proposition `; neutral encyclopedic overview. | -| Vote Smart | `votesmart.ts` | `vote_smart` | State-level measures. Fuzzy-matches the title to a Vote Smart measure → summary, full-text URL, pro/con URLs. Requires `VOTE_SMART_API_KEY`. | -| Google Civic | (the input itself) | `google_civic` | The measure as the API returned it — lowest-trust, so its subtitle/text still surface when nothing better exists. | +| Source | Adapter | Tier | Scope / method | +| -------------------------------- | ---------------------- | ----------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| CA SOS Official Voter Guide | `ca-sos-voterguide.ts` | `state_sos` | CA statewide props. Official AG summary, pro/con, full-text URL — **real official text, not AI.** Matches on the prop number parsed from the title. The guide is rebuilt each cycle and only serves the active election, so it yields nothing between prop cycles. | +| CA LAO Fiscal Analyses | `ca-lao-fiscal.ts` | `state_sos` | CA statewide props. Official nonpartisan fiscal impact analysis for each proposition. HTML scrape of `lao.ca.gov/BallotAnalysis/Proposition?number=N&year=YYYY` — no API exists. Pre-warmed by the `ca-lao-fiscal` scraper into `CivicApiCache`; adapter reads cache first, falls back to live fetch. Only fires for parsed proposition numbers (not local lettered measures). | +| Texas SOS + Legislative Council | `texas-official.ts` | `state_sos` + `legislative_council` | Current-cycle Texas statewide propositions. Matches Google Civic/Vote Smart-style records by election date/year plus proposition number (title similarity fallback), then supplies separately cited SOS result facts and TLC page-cited explanation. | +| League of Women Voters — CaVotes | `cavotes.ts` | `lwv` | CA statewide props. Nonpartisan "Pros & Cons" summary, fiscal effects, supporter/opponent arguments via the CaVotes WordPress REST API (`cavotes.org/wp-json/wp/v2/ballots`). Slugs are inconsistent across years, so it enumerates the list and matches by prop number + year. | +| Ballotpedia | `ballotpedia.ts` | `ballotpedia` | **Statewide _and_ local** lettered measures — the main source for local. Rendered article HTML (MediaWiki API disabled), resolved from a year/county index. Extracts ballot summary/question, fiscal impact / impartial analysis, arguments. Year-gated so a same-letter measure from another cycle isn't surfaced. | +| Wikipedia | `wikipedia.ts` | `wikipedia` | CA statewide props only — gated on a parsed prop number (local titles like "Measure Q" collide with unrelated articles). MediaWiki extract for ` California Proposition `; neutral encyclopedic overview. | +| Vote Smart | `votesmart.ts` | `vote_smart` | State-level measures. Fuzzy-matches the title to a Vote Smart measure → summary, full-text URL, pro/con URLs. Requires `VOTE_SMART_API_KEY`. | +| Google Civic | (the input itself) | `google_civic` | The measure as the API returned it — lowest-trust, so its subtitle/text still surface when nothing better exists. | **Local lettered measures:** the County Counsel / City Attorney **Impartial Analysis** (carried on Ballotpedia) is extracted on its own and wins the summary slot ahead of the bare ballot question — it's the authoritative neutral text, so it no longer gets buried in the fiscal field, and the advocacy/AI fallback only runs when it's absent. @@ -100,6 +102,7 @@ flowchart TD | `summaryIsAiGenerated` | true if `summary` came from AI | | `fiscalImpact` | official fiscal analysis | | `proArguments[]` / `conArguments[]` | attributed arguments (`{text, author?, sourceName, sourceUrl}`) | +| `result` | official SOS status, outcome, vote totals, choices, as-of time, and source link | | `citations[]` | which source supplied each field (`{field, sourceName, sourceUrl, tier, official}`) | | `sources[]` | one `Source` per citation, for attribution chips | diff --git a/docs/ncsbe-election-data.md b/docs/ncsbe-election-data.md new file mode 100644 index 00000000..287932b8 --- /dev/null +++ b/docs/ncsbe-election-data.md @@ -0,0 +1,49 @@ +# NCSBE current-cycle election data + +The `ncsbe` scraper discovers official public files from the NCSBE candidate-list +and election-results index pages. It does not hard-code election file URLs. At +runtime, discovery accepts only links for the current calendar-year election +cycle and prefers the structured candidate CSV and result ZIP/TSV files. The +official referendum PDF is used because NCSBE does not publish a structured +referendum download. + +```bash +pnpm --filter @acme/scraper run start ncsbe --max-items 4 +``` + +## Data boundaries + +- Candidate addresses, phone numbers, and email addresses are discarded by the + parser and have no database columns. +- Voter registration and voter-history products are never discovered or fetched. +- Historical links may be represented by small parser regression fixtures, but + runtime discovery, persistence, and API reads reject non-current-cycle dates. +- Durham County is the primary validation jurisdiction. Wake County fixtures + prevent county-specific assumptions. + +## Storage and idempotency + +`election_source` stores provider, source kind, exact URL, fetched time, SHA-256, +parser structure version, election date, and certification state. Provider-neutral +candidate, referendum, and result tables reference that row. A re-run upserts the +same source identity and replaces its child rows in one transaction, so interrupted +runs roll back and completed re-runs do not duplicate records. + +The result ZIP's official media file is marked `official_not_certified`. NCSBE's +separate certified-result PDFs are intentionally not used when structured data is +available; a future certified structured file can use `certified` without changing +the reader contract. + +## API reader and Civic matching + +`civic.getNcElectionData` accepts `county`, an ISO `electionDate`, optional Google +Civic contest references, and optional `includePrecincts`. It returns candidate, +referendum, county-total result, and (when requested) precinct records. Every row +includes its exact source URL, checksum, fetched time, structure version, and +certification state. + +Contest and candidate matching first compares normalized values exactly (case, +punctuation, `NC`/`North Carolina`, and primary-party suffixes are normalized). +The only fuzzy fallback is token Dice similarity of at least `0.82`, and it is +accepted only when the best result is at least `0.08` ahead of the runner-up. +Uncertain matches are omitted rather than guessed. diff --git a/docs/scraper.md b/docs/scraper.md index fb55da25..71118888 100644 --- a/docs/scraper.md +++ b/docs/scraper.md @@ -15,33 +15,72 @@ process environment at runtime, not embedded during the build. ## Scrapers -| Scraper | Source | Content type | Method | -| ---------------------- | ------------------------------ | -------------------- | ------------------------------------------------------ | -| `congress.ts` | congress.gov REST API | `bill` | REST (`CONGRESS_API_KEY`), incremental by `updateDate` | -| `federalregister.ts` | federalregister.gov REST API | `government_content` | REST; HTML→Markdown via Turndown | -| `scotus.ts` | CourtListener REST API | `court_case` | REST (`COURTLISTENER_API_KEY`, optional) | -| `vote411.ts` | vote411.org | (cached locally) | cheerio HTML parse; does **not** write to the main DB | -| `scc-cvig.ts` | Santa Clara County voter guide | `civic_api_cache` | PDF extraction; optional Gemini fallback | -| `ca-sos-statements.ts` | CA Secretary of State guide | `civic_api_cache` | official candidate-statement pages | -| `ca-lao-fiscal.ts` | CA LAO ballot analyses | `civic_api_cache` | proposition fiscal analyses via HTML parse | -| `ca-vig-archive.ts` | CA SOS voter-guide archive | `civic_api_cache` | historical proposition guide pages via HTML parse | +| Scraper | Source | Content type | Method | +| --------------------------- | -------------------------------- | -------------------------- | ----------------------------------------------------------------------- | +| `congress.ts` | congress.gov REST API | `bill` | REST (`CONGRESS_API_KEY`), incremental by `updateDate` | +| `federalregister.ts` | federalregister.gov REST API | `government_content` | REST; HTML→Markdown via Turndown | +| `scotus.ts` | CourtListener REST API | `court_case` | REST (`COURTLISTENER_API_KEY`, optional) | +| `vote411.ts` | vote411.org | (cached locally) | cheerio HTML parse; does **not** write to the main DB | +| `scc-cvig.ts` | Santa Clara County voter guide | `civic_api_cache` | PDF extraction; optional Gemini fallback | +| `ca-sos-statements.ts` | CA Secretary of State guide | `civic_api_cache` | official candidate-statement pages | +| `ca-lao-fiscal.ts` | CA LAO ballot analyses | `civic_api_cache` | proposition fiscal analyses via HTML parse | +| `ca-vig-archive.ts` | CA SOS voter-guide archive | `civic_api_cache` | historical proposition guide pages via HTML parse | +| `texas-current-election.ts` | Texas SOS + TLC | `election_source_snapshot` | current-cycle JSON + deterministic PDF text parsing | +| `texas-legislature.ts` | Texas Legislative Council FTP | `bill` | current-session XML + bulk HTML; no site mining | +| `civicengage.ts` | Cedar Park official council page | local-government tables | CivicEngage entry page + Municode embed; deterministic HTML/PDF parsing | +| `durham-onbase.ts` | Durham OnBase Agenda Online | local-government tables | current-cycle meetings, items, attachments, and official actions | +| `durham-bocc.ts` | Durham County Legistar API | local-government tables | current-cycle meetings, items, actions, votes, and documents | All HTTP goes through one `fetchWithRetry()` utility (`apps/scraper/src/utils/fetch.ts`): exponential backoff (1s/2s/4s…), `Retry-After` support (seconds or HTTP-date), 30s default timeout via `AbortController`, retriable on 429/5xx and `ECONNRESET`/`ECONNREFUSED`, plus a stateful **per-host backoff** that ramps on 429/5xx and relaxes on success. > Note: `whitehouse.gov` cheerio scraping was replaced by the structured **Federal Register** REST API. `vote411-ballot.ts` exists for address-based ballot lookup (needs Playwright) but isn't wired into the CLI. +The Texas scraper is intentionally current-cycle only. SOS election facts and +TLC explanatory text remain separate provider snapshots and separate citations; +see [Texas current-election data](./texas-current-election.md). + +The Cedar Park pilot is deliberately limited to City Council meetings from the +latest 12 months. It stores meetings, versioned documents, agenda items, +motions/outcomes, and named roll-call votes in provider-neutral tables. It does +not browse or backfill historical election cycles. + +### Durham OnBase + +Run `pnpm -F @acme/scraper start:dev -- durham-onbase`. The scraper reads the +official OnBase embedded meeting JSON, agenda/minutes HTML outlines, and agenda +item detail endpoints. It does not use AI or PDF vision. Requests are serialized +at a minimum 250 ms interval, use the shared retry/backoff client, and skip rows +refreshed in the last 24 hours by default. + +Only meetings in the current calendar-year council cycle are ingested. The +product does not expose historical election cycles or run an OnBase backfill. +`DURHAM_ONBASE_MAX_ITEMS` (default `100`) limits meetings and +`DURHAM_ONBASE_CACHE_TTL_HOURS` (default `24`) controls refresh frequency. + +### Durham County BOCC + +`durham-bocc` reads Durham County's official structured Legistar feed for BOCC body ID `138`. It bounds discovery to the current two-year election cycle, upserts stable provider IDs, and stores checksums/source row versions so replaced agendas update the existing meeting. Cancellations and explicit amendments remain visible; Spanish attachments are tagged separately. Agenda/minutes PDFs are retained as official links and are not sent through AI or OCR when structured item/action fields are available. + +Run it with `pnpm --filter @acme/scraper run start durham-bocc`. The default cap is 100 meetings and can be changed with `DURHAM_BOCC_MAX_ITEMS` or `--max-items`. + ## Upsert + Change Detection `apps/scraper/src/utils/db/operations.ts` centralizes writes behind a discriminated-union `upsertContent(type, data)` (`type` ∈ bill | government_content | court_case). Each run: 1. Compute a SHA-256 over the type-specific key fields (title, summary, full text, status…). -2. Look up the existing row by its natural key (`(billNumber, sourceWebsite)`, `url`, or `caseNumber`). +2. Look up the existing row by its natural key (`(billNumber, sourceWebsite, legislativeSession)`, `url`, or `caseNumber`). 3. **Unchanged hash** → skip AI entirely; backfill only missing AI assets. 4. **New bill without a source description** → generate the required description first; defer the insert if source text or every provider is unavailable. 5. **New or changed** → run the remaining AI pipeline, upsert via `onConflictDoUpdate`, append to `versions`. `SCRAPER_FORCE_AI_REGEN=1` overrides the cache. A `isUsableText()` gate refuses to feed AI any text under 200 chars or that's mostly blank/all-caps/single-word lines — keeps the model from "summarizing" garbage. +Texas is the provider-neutral state-bill extension to this flow. Its rows carry +an OCD jurisdiction, legislative session, subjects, sponsorships, documents, +votes, and an optional exact Open States ID. The scraper selects only the latest +FTP session and sets `skipEnrichment`; the API exposes only that newest session +through `content.texasBills`, with full supporting material in `content.getById`. + ## AI Pipeline Provider config lives in `apps/scraper/src/utils/ai/provider.ts`: text uses **OpenRouter** first, then an OpenAI-compatible local endpoint (`LOCAL_LLM_BASE_URL`, such as Ollama), with direct DeepSeek retained only as a deprecated last resort. PDF vision fallback uses **Gemini `gemini-2.5-flash`**. Images use hosted **Black Forest Labs FLUX.2 Klein 9B**, then `LOCAL_FLUX_BASE_URL` as a local fallback. Provider usage and hosted-image costs are tracked per run. diff --git a/docs/texas-current-election.md b/docs/texas-current-election.md new file mode 100644 index 00000000..5055cfed --- /dev/null +++ b/docs/texas-current-election.md @@ -0,0 +1,85 @@ +# Texas Current-Election Data + +## Scope + +`texas-current-election` ingests **only the current Texas election cycle** plus +the latest cycle-specific constitutional-amendment analysis. It does not expose +historical browsing, accept a year from API callers, or backfill older cycles. + +The Texas Secretary of State (SOS) is authoritative for statewide election +facts: elections, contests, candidates, vote totals, county totals, turnout, +reporting progress, outcomes, and whether the results application labels an +election official. The Texas Legislative Council (TLC) supplies nonpartisan +explanatory text: ballot language, summary/background analysis, supporter and +opponent comments, and fiscal implications. These providers are persisted in +separate rows and receive separate field-level citations in the reader. + +Texas SOS is **not a complete source for local Texas elections**. This pipeline +normalizes the statewide, federal, and district records the SOS publishes; it +does not imply that every city, school-district, county, or special-district +contest is present. + +## Sources and discovery + +- SOS results app: `https://goelect.txelections.civixapps.com/ivis-enr-ui/races` +- TLC publications: `https://tlc.texas.gov/publications` + +The scraper reads the SOS application's `electionConstants` JSON to discover +the newest cycle and its election IDs. It never constructs a URL from a +hard-coded year. It also discovers the newest non-condensed `analysesNN.pdf` +link from the TLC publications HTML. When the TLC report belongs to the prior +calendar year (for example, the 2025 amendment election during the 2026 general +cycle), the scraper fetches only that matching SOS constitutional-amendment +election so the current TLC analysis can be paired with its SOS outcome. + +The Civix API wraps sections as base64 JSON. Parsing is deterministic and +normalizes: + +- candidate and proposition contests, parties, incumbency, votes, percentages, + early votes, winners, and proposition outcomes; +- counties/precincts/polling places reporting and official/unofficial status; +- per-county choice totals and turnout; +- source version and SHA-256 content checksum. + +TLC PDFs are read from their text layer page-by-page. There is no AI or vision +step in the normal path. Each extracted field links to `#page=N` in the official +PDF. Missing optional sections produce diagnostics and do not discard the rest +of the proposition. + +## Persistence and reader + +The provider-neutral `election_source_snapshot` table stores one idempotent row +per `(jurisdiction, cycle_year, provider, scope)`. Texas writes `texas-sos` and +`texas-tlc` rows with `scope = current`; certification/status refreshes update +those rows instead of creating duplicates. + +Apply `packages/db/migrations/add_election_source_snapshots.sql` before the +first run, then execute: + +```sh +pnpm --filter @acme/scraper run start texas-current-election +``` + +`TX_SOS_MAX_ITEMS` (default `12`) or the CLI `--max-items` flag caps source +records per run. `POSTGRES_URL` is the only required credential; both sources +are public and require no API key. + +Consumers call the public tRPC query `civic.getTexasCurrentElection`. It has no +input by design. The response contains the newest SOS cycle and the latest TLC +amendment cycle. TLC propositions are matched to SOS outcomes by election cycle +and proposition number; each result retains its SOS citation while every +analysis field retains its TLC page citation. + +The same persisted data is wired into `civic.getVoterInfo` through +`measure-sources/texas-official.ts`. A Google Civic measure is matched by exact +election date/year and proposition number, with title similarity only as a +fallback when the number is absent. Vote Smart matching also rejects records +whose published election date differs. The canonical measure exposes SOS +`result` facts and TLC summary/fiscal/pro-con fields with separate citations. + +## Verification + +Fixtures cover candidate results, county amendment totals, the 2025 TLC layout, +and the alternate 2023 layout. Tests assert current-cycle discovery, idempotent +identities, official status, turnout/outcomes, page citations, source separation, +and fail-soft behavior for a missing section. diff --git a/packages/api/package.json b/packages/api/package.json index 661eb542..2da34124 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -11,6 +11,10 @@ "types": "./dist/integrations/legistar.d.ts", "default": "./src/integrations/legistar.ts" }, + "./lib/local-government": { + "types": "./dist/lib/local-government.d.ts", + "default": "./src/lib/local-government.ts" + }, "./lib/civic": { "types": "./dist/lib/civic.d.ts", "default": "./src/lib/civic.ts" @@ -47,6 +51,10 @@ "types": "./dist/lib/measure-sources/types.d.ts", "default": "./src/lib/measure-sources/types.ts" }, + "./lib/texas-election-data": { + "types": "./dist/lib/texas-election-data.d.ts", + "default": "./src/lib/texas-election-data.ts" + }, "./lib/measure-sources/disabled/vig-archive": { "types": "./dist/lib/measure-sources/disabled/vig-archive.d.ts", "default": "./src/lib/measure-sources/disabled/vig-archive.ts" diff --git a/packages/api/src/index.ts b/packages/api/src/index.ts index 0d729333..97970162 100644 --- a/packages/api/src/index.ts +++ b/packages/api/src/index.ts @@ -39,6 +39,15 @@ export type { } from "./lib/elected-officials"; export { getElectedOfficials } from "./lib/elected-officials"; +export type { + CivicContestReference, + ElectionMatch, +} from "./lib/ncsbe-election-data"; +export { + getCurrentNcElectionData, + matchNcsbeName, +} from "./lib/ncsbe-election-data"; + // California Secretary of State live election-results feed export type { ElectionContestResult, @@ -88,6 +97,13 @@ export { LegistarError, JURISDICTIONS, } from "./integrations/legistar"; + +// Provider-neutral local-government records persisted by source scrapers. +export { + getLocalGovernmentMeeting, + getLocalGovernmentMeetings, +} from "./lib/local-government"; +export type { LocalGovernmentMeetingQuery } from "./lib/local-government"; export type { Jurisdiction, LegistarMeeting, diff --git a/packages/api/src/lib/__fixtures__/texas-sos-amendment.json b/packages/api/src/lib/__fixtures__/texas-sos-amendment.json new file mode 100644 index 00000000..e730f3d3 --- /dev/null +++ b/packages/api/src/lib/__fixtures__/texas-sos-amendment.json @@ -0,0 +1,5 @@ +{ + "Version": "enr/51031/86/", + "Home": "eyJFbGVjRGF0ZSI6IjExMDQyMDI1IiwiQ291bnRpZXNSZXBvcnRpbmciOnsiQ1IiOjI1NCwiQ1QiOjI1NH0sIkxhc3RVcGRhdGVkVGltZSI6Ik5vdiAwNiwgMjAyNSAwNDoxODoyMyIsIlByZWNpbmN0c1JlcG9ydGluZyI6eyJQUiI6MTE3MiwiUFQiOjExNzJ9LCJQb2xsaW5nUmVwb3J0aW5nIjp7IlBMUiI6NDYzNywiUExUIjo0NjM3fX0=", + "Race": "eyJPZmZpY2VUeXBlcyI6W3siT2ZmaWNlVHlwZSI6IlNUQVRFV0lERSBQUk9QT1NJVElPTlMiLCJSYWNlcyI6W3siaWQiOjg3LCJOIjoiUFJPUE9TSVRJT04gMSAoU0pSIDU5KSIsIkNhbmRpZGF0ZXMiOlt7Ik4iOiJGT1IiLCJWIjoyMDQxODU5LCJQRSI6NjkuMDMsIkVWIjoxMDA4MzkxfSx7Ik4iOiJBR0FJTlNUIiwiViI6OTE2MjE3LCJQRSI6MzAuOTcsIkVWIjo0NTY1Mzd9XSwiVCI6Mjk1ODA3Nn1dfV19" +} diff --git a/packages/api/src/lib/__fixtures__/texas-sos-constants.json b/packages/api/src/lib/__fixtures__/texas-sos-constants.json new file mode 100644 index 00000000..a42bddc9 --- /dev/null +++ b/packages/api/src/lib/__fixtures__/texas-sos-constants.json @@ -0,0 +1,3 @@ +{ + "upload": "eyJlbGVjdGlvbkluZm8iOnsiMjAyNiI6eyJQIjp7IjUzODEzIjp7IklEIjo1MzgxMywiTiI6IjIwMjYgUkVQVUJMSUNBTiBQUklNQVJZIEVMRUNUSU9OIiwiTyI6IlkiLCJZIjoiMjAyNiIsIkVUIjoiUCIsIkVDIjoiU1cifX0sIkdFIjp7IjUzODE1Ijp7IklEIjo1MzgxNSwiTiI6IjIwMjYgTk9WRU1CRVIgR0VORVJBTCBFTEVDVElPTiIsIk8iOiJOIiwiWSI6IjIwMjYiLCJFVCI6IkdFIiwiRUMiOiJTVyJ9fX0sIjIwMjUiOnsiUyI6eyI1MTAzMSI6eyJJRCI6NTEwMzEsIk4iOiIyMDI1IE5PVkVNQkVSIDRUSCBDT05TVElUVVRJT05BTCBBTUVORE1FTlQiLCJPIjoiTiIsIlkiOiIyMDI1IiwiRVQiOiJTIiwiRUMiOiJTVyJ9fX19fQ==" +} diff --git a/packages/api/src/lib/__fixtures__/texas-sos-county.json b/packages/api/src/lib/__fixtures__/texas-sos-county.json new file mode 100644 index 00000000..e6d5e5fb --- /dev/null +++ b/packages/api/src/lib/__fixtures__/texas-sos-county.json @@ -0,0 +1,3 @@ +{ + "upload": "eyIxIjp7Ik4iOiJBTkRFUlNPTiIsIlRWIjo4MDU0NCwiUmFjZXMiOnsiODciOnsiT0lEIjo4NywiTiI6IlBST1BPU0lUSU9OIDEgKFNKUiA1OSkiLCJDIjp7Ijg3MDEiOnsiaWQiOjg3MDEsIk4iOiJGT1IiLCJWIjozMzYxLCJQRSI6NzAuOTIsIkVWIjoxNTAyfSwiODcwMiI6eyJpZCI6ODcwMiwiTiI6IkFHQUlOU1QiLCJWIjoxMzc4LCJQRSI6MjkuMDgsIkVWIjo2MDl9fSwiUFIiOjIzLCJUUCI6MjN9fSwiU3VtbWFyeSI6eyJSViI6MzA1NzksIlZDIjo0NzM5LCJWVCI6MTUuNX19fQ==" +} diff --git a/packages/api/src/lib/__fixtures__/texas-sos-election.json b/packages/api/src/lib/__fixtures__/texas-sos-election.json new file mode 100644 index 00000000..af41c508 --- /dev/null +++ b/packages/api/src/lib/__fixtures__/texas-sos-election.json @@ -0,0 +1,5 @@ +{ + "Version": "enr/53813/135/", + "Home": "eyJFbGVjRGF0ZSI6IjAzMDMyMDI2IiwiQ291bnRpZXNSZXBvcnRpbmciOnsiQ1IiOjI1NCwiQ1QiOjI1NH0sIkxhc3RVcGRhdGVkVGltZSI6IkFwciAxNCwgMjAyNiAyMDo0ODoxNyIsIlByZWNpbmN0c1JlcG9ydGluZyI6eyJQUiI6NSwiUFQiOjV9LCJQb2xsaW5nUmVwb3J0aW5nIjp7IlBMUiI6NDYzNiwiUExUIjo0NjM2fX0=", + "Race": "eyJPZmZpY2VUeXBlcyI6W3siT2ZmaWNlVHlwZSI6IlNUQVRFV0lERSBPRkZJQ0VTIiwiUmFjZXMiOlt7ImlkIjozNTc5LCJOIjoiR09WRVJOT1IiLCJDYW5kaWRhdGVzIjpbeyJJRCI6MjUyNzMsIk4iOiJHUkVHIEFCQk9UVCAoSSkiLCJQIjoiUkVQIiwiViI6MTc2NDkyNCwiUEUiOjgxLjgzLCJFViI6MTEzMzEyNX0seyJJRCI6MzI4MTUsIk4iOiJSLkYuIFwiQk9CXCIgQUNIR0lMTCIsIlAiOiJSRVAiLCJWIjo5MTE5LCJQRSI6MC40MiwiRVYiOjU2MTd9XSwiVCI6MTc3NDA0M31dfV19" +} diff --git a/packages/api/src/lib/__fixtures__/texas-tlc-2023.json b/packages/api/src/lib/__fixtures__/texas-tlc-2023.json new file mode 100644 index 00000000..53f891fd --- /dev/null +++ b/packages/api/src/lib/__fixtures__/texas-tlc-2023.json @@ -0,0 +1,10 @@ +[ + { + "page": 1, + "text": "Analyses of Proposed Constitutional Amendments November 7, 2023, Election Texas Legislative Council" + }, + { + "page": 9, + "text": "PROPOSITION 1 (H.J.R. 126) The constitutional amendment protecting the right to engage in farming. SUMMARY ANALYSIS The proposal protects generally accepted agricultural practices. BACKGROUND Existing statutes regulate agricultural operations. SUMMARY OF COMMENTS Comments by Supporters: • The amendment would protect farms from unnecessary regulation. Text of H.J.R. 126" + } +] diff --git a/packages/api/src/lib/__fixtures__/texas-tlc-2025.json b/packages/api/src/lib/__fixtures__/texas-tlc-2025.json new file mode 100644 index 00000000..13cedbeb --- /dev/null +++ b/packages/api/src/lib/__fixtures__/texas-tlc-2025.json @@ -0,0 +1,22 @@ +[ + { + "page": 1, + "text": "ANALYSES OF PROPOSED CONSTITUTIONAL AMENDMENTS 89TH TEXAS LEGISLATURE NOVEMBER 4, 2025, ELECTION Published by the Texas Legislative Council August 2025" + }, + { + "page": 10, + "text": "Proposition 1 (S.J.R. 59) The constitutional amendment providing for the creation of the permanent technical institution infrastructure fund. SUMMARY ANALYSIS S.J.R. 59 proposes an amendment creating two funds for technical education. The legislature appropriated $850 million to the permanent fund. BACKGROUND AND DETAILED ANALYSIS Existing law provides general funding to public institutions. The new fund would provide a dedicated source of revenue." + }, + { + "page": 11, + "text": "SUMMARY OF COMMENTS Comments by Supporters: • The amendment would expand workforce training capacity. • A dedicated fund would improve long-term planning. Comments by Opponents: • A perpetual fund would reduce the discretion of future legislatures." + }, + { + "page": 12, + "text": "Text of S.J.R. 59 SENATE JOINT RESOLUTION proposing a constitutional amendment." + }, + { + "page": 14, + "text": "Proposition 2 (S.J.R. 18) The constitutional amendment prohibiting a tax on capital gains. SUMMARY ANALYSIS The resolution prohibits a tax. BACKGROUND AND DETAILED ANALYSIS Texas does not currently impose this tax. SUMMARY OF COMMENTS Comments by Supporters: • The prohibition gives taxpayers certainty. Comments by Opponents: • The constitution should not contain tax policy. Text of S.J.R. 18" + } +] diff --git a/packages/api/src/lib/civic.ts b/packages/api/src/lib/civic.ts index b268ea16..c39b2199 100644 --- a/packages/api/src/lib/civic.ts +++ b/packages/api/src/lib/civic.ts @@ -16,6 +16,7 @@ import type { StatewideOffice, } from "../clients/ca-sos-results"; import type { CrossValidateContext } from "./measure-crossvalidate"; +import type { MeasureResult } from "./measure-sources/types"; import { getDistrictResults, getStatewideResults, @@ -217,6 +218,8 @@ export interface Contest { conArguments?: MeasureArgumentRef[]; /** Per-field source attribution for everything above. */ citations?: MeasureCitationRef[]; + /** Official election result facts when an SOS match is available. */ + result?: MeasureResult; } export interface Candidate { @@ -472,6 +475,7 @@ interface EnrichmentContext { stateAbbrev?: string; county?: string; electionYear?: number; + electionDate?: string; } /** @@ -514,6 +518,7 @@ async function enrichContest( stateAbbrev: ctx?.stateAbbrev, county: ctx?.county, electionYear: ctx?.electionYear ?? new Date().getFullYear(), + electionDate: ctx?.electionDate, }; try { @@ -543,6 +548,7 @@ async function enrichContest( contest.citations = merged.citations.length ? merged.citations : undefined; + contest.result = merged.result; // Back-fill the legacy single-field shape so existing UI keeps working. if (!contest.referendumText && merged.fullText) { @@ -838,6 +844,7 @@ export async function getVoterInfo( electionYear: resp.election.electionDay ? new Date(resp.election.electionDay).getFullYear() : new Date().getFullYear(), + electionDate: resp.election.electionDay, }); const params: Record = { address }; diff --git a/packages/api/src/lib/local-government.ts b/packages/api/src/lib/local-government.ts new file mode 100644 index 00000000..e3bb35c5 --- /dev/null +++ b/packages/api/src/lib/local-government.ts @@ -0,0 +1,124 @@ +import { and, asc, desc, eq, gte, inArray, lte } from "@acme/db"; +import { db } from "@acme/db/client"; +import { + LocalGovernmentAgendaItem, + LocalGovernmentDocument, + LocalGovernmentMeeting, + LocalGovernmentVote, +} from "@acme/db/schema"; + +export interface LocalGovernmentMeetingQuery { + jurisdiction?: string; + start?: Date; + end?: Date; + limit?: number; +} + +export async function getLocalGovernmentMeetings( + query: LocalGovernmentMeetingQuery = {}, +) { + const filters = []; + if (query.jurisdiction) { + filters.push(eq(LocalGovernmentMeeting.jurisdiction, query.jurisdiction)); + } + if (query.start) + filters.push(gte(LocalGovernmentMeeting.startsAt, query.start)); + if (query.end) filters.push(lte(LocalGovernmentMeeting.startsAt, query.end)); + + const meetings = await db + .select() + .from(LocalGovernmentMeeting) + .where(filters.length > 0 ? and(...filters) : undefined) + .orderBy(desc(LocalGovernmentMeeting.startsAt)) + .limit(Math.min(query.limit ?? 50, 100)); + if (meetings.length === 0) return []; + + const documents = await db + .select({ + id: LocalGovernmentDocument.id, + meetingId: LocalGovernmentDocument.meetingId, + type: LocalGovernmentDocument.type, + title: LocalGovernmentDocument.title, + url: LocalGovernmentDocument.url, + }) + .from(LocalGovernmentDocument) + .where( + inArray( + LocalGovernmentDocument.meetingId, + meetings.map((meeting) => meeting.id), + ), + ) + .orderBy(asc(LocalGovernmentDocument.type)); + + return meetings.map((meeting) => ({ + ...meeting, + documents: documents.filter( + (document) => document.meetingId === meeting.id, + ), + })); +} + +export async function getLocalGovernmentMeeting(id: string) { + const [meeting] = await db + .select() + .from(LocalGovernmentMeeting) + .where(eq(LocalGovernmentMeeting.id, id)) + .limit(1); + if (!meeting) return null; + + const [documents, items] = await Promise.all([ + db + .select({ + id: LocalGovernmentDocument.id, + meetingId: LocalGovernmentDocument.meetingId, + type: LocalGovernmentDocument.type, + title: LocalGovernmentDocument.title, + url: LocalGovernmentDocument.url, + mediaType: LocalGovernmentDocument.mediaType, + checksum: LocalGovernmentDocument.checksum, + isCurrent: LocalGovernmentDocument.isCurrent, + discoveredAt: LocalGovernmentDocument.discoveredAt, + fetchedAt: LocalGovernmentDocument.fetchedAt, + }) + .from(LocalGovernmentDocument) + .where(eq(LocalGovernmentDocument.meetingId, meeting.id)) + .orderBy( + desc(LocalGovernmentDocument.isCurrent), + asc(LocalGovernmentDocument.type), + ), + db + .select() + .from(LocalGovernmentAgendaItem) + .where(eq(LocalGovernmentAgendaItem.meetingId, meeting.id)) + .orderBy(asc(LocalGovernmentAgendaItem.sequence)), + ]); + + const votes = + items.length === 0 + ? [] + : await db + .select() + .from(LocalGovernmentVote) + .where( + inArray( + LocalGovernmentVote.agendaItemId, + items.map((item) => item.id), + ), + ) + .orderBy(asc(LocalGovernmentVote.voterName)); + const votesByItem = new Map(); + for (const vote of votes) { + const list = votesByItem.get(vote.agendaItemId) ?? []; + list.push(vote); + votesByItem.set(vote.agendaItemId, list); + } + + return { + ...meeting, + documents, + items: items.map((item) => ({ + ...item, + votes: votesByItem.get(item.id) ?? [], + })), + }; +} diff --git a/packages/api/src/lib/measure-crossvalidate.ts b/packages/api/src/lib/measure-crossvalidate.ts index 1fd95ece..15a5e63d 100644 --- a/packages/api/src/lib/measure-crossvalidate.ts +++ b/packages/api/src/lib/measure-crossvalidate.ts @@ -23,6 +23,7 @@ import { enrichFromLao } from "./measure-sources/ca-lao-fiscal"; import { enrichFromCaSos } from "./measure-sources/ca-sos-voterguide"; import { enrichFromCaVotes } from "./measure-sources/cavotes"; import { collectGroundingText } from "./measure-sources/grounded-fallback"; +import { enrichFromTexasOfficial } from "./measure-sources/texas-official"; import { SOURCE_TIER_RANK } from "./measure-sources/types"; import { enrichFromWikipedia } from "./measure-sources/wikipedia"; import { enrichFromVoteSmart } from "./votesmart"; @@ -31,6 +32,7 @@ export interface CrossValidateContext { stateAbbrev?: string; county?: string; electionYear: number; + electionDate?: string; } /** A measure as Google Civic gives it to us — the lowest-tier source. */ @@ -55,6 +57,7 @@ async function collectVoteSmart( title, ctx.stateAbbrev, ctx.electionYear, + ctx.electionDate, ).catch(() => null); if (!vs) return null; return { @@ -142,7 +145,7 @@ export async function crossValidateMeasure( input: CivicMeasureInput, ctx: CrossValidateContext, ): Promise { - const [sos, lwv, bp, wiki, vs, lao] = await Promise.all([ + const [sos, lwv, bp, wiki, vs, lao, texas] = await Promise.all([ enrichFromCaSos(input.title, ctx.stateAbbrev, ctx.electionYear).catch( () => null, ), @@ -161,6 +164,9 @@ export async function crossValidateMeasure( enrichFromLao(input.title, ctx.stateAbbrev, ctx.electionYear).catch( () => null, ), + enrichFromTexasOfficial(input.title, ctx).catch( + (): { sos?: MeasureSourceData; tlc?: MeasureSourceData } => ({}), + ), ]); const sources: MeasureSourceData[] = [ @@ -170,6 +176,8 @@ export async function crossValidateMeasure( wiki, vs, lao, + texas.sos ?? null, + texas.tlc ?? null, civicAsSource(input), ].filter((s): s is MeasureSourceData => s !== null); sources.sort(byTierDesc); @@ -217,6 +225,11 @@ export async function crossValidateMeasure( } } + // --- Official result facts: SOS only; explanatory sources leave absent. --- + const resultSource = sources.find((source) => source.result); + const result = resultSource?.result; + if (resultSource && result) citations.push(cite("result", resultSource)); + // --- Pro / con arguments: collect from all sources, dedupe, attribute. --- const proArguments = collectArguments(sources, "pro", citations); const conArguments = collectArguments(sources, "con", citations); @@ -381,6 +394,7 @@ export async function crossValidateMeasure( fullTextUrl, proArguments, conArguments, + result, citations, discrepancies: discrepancies.length ? discrepancies : undefined, }; diff --git a/packages/api/src/lib/measure-sources/texas-official.test.ts b/packages/api/src/lib/measure-sources/texas-official.test.ts new file mode 100644 index 00000000..7f177eb1 --- /dev/null +++ b/packages/api/src/lib/measure-sources/texas-official.test.ts @@ -0,0 +1,111 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import type { TexasCurrentElectionData } from "../texas-election-data"; +import { matchTexasOfficialMeasure } from "./texas-official"; + +const current: TexasCurrentElectionData = { + jurisdiction: "TX", + cycleYear: 2026, + elections: [], + fetchedAt: "2026-07-21T00:00:00.000Z", + diagnostics: [], + constitutionalAmendments: { + cycleYear: 2025, + electionDate: "November 4, 2025", + citation: { + sourceName: "Texas Legislative Council", + sourceUrl: "https://tlc.texas.gov/docs/amendments/analyses25.pdf", + provider: "texas-tlc", + official: true, + }, + measures: [ + { + propositionNumber: 1, + ballotLanguage: + "The constitutional amendment creating a permanent technical institution infrastructure fund.", + summaryAnalysis: "Creates a dedicated education infrastructure fund.", + supporterArguments: ["Supporters say the fund expands training."], + opponentArguments: ["Opponents question a permanent appropriation."], + fiscalImplications: ["The legislature appropriated $850 million."], + pageStart: 10, + pageEnd: 17, + diagnostics: [], + citations: { + summaryAnalysis: { + sourceName: "Texas Legislative Council", + sourceUrl: + "https://tlc.texas.gov/docs/amendments/analyses25.pdf#page=10", + provider: "texas-tlc", + official: true, + page: 10, + }, + }, + result: { + status: "complete", + outcome: "adopted", + totalVotes: 100, + choices: [ + { + name: "FOR", + incumbent: false, + votes: 70, + percent: 70, + winner: true, + }, + { + name: "AGAINST", + incumbent: false, + votes: 30, + percent: 30, + winner: false, + }, + ], + citation: { + sourceName: "Texas Secretary of State", + sourceUrl: + "https://goelect.txelections.civixapps.com/ivis-enr-ui/races", + provider: "texas-sos", + official: true, + }, + }, + }, + ], + }, +}; + +void test("matches Google/Vote Smart-style titles by date and proposition number", () => { + const matched = matchTexasOfficialMeasure( + "Texas Proposition 1, Technical Education Fund Amendment", + { + stateAbbrev: "TX", + electionYear: 2025, + electionDate: "2025-11-04", + }, + current, + ); + const amendment = current.constitutionalAmendments; + assert.ok(amendment); + assert.ok(matched.sos); + assert.ok(matched.tlc); + assert.equal(matched.sos.result?.outcome, "adopted"); + assert.equal( + matched.tlc.officialSummary, + amendment.measures[0]?.summaryAnalysis, + ); + assert.equal(matched.sos.sourceName, "Texas Secretary of State"); + assert.equal(matched.tlc.sourceName, "Texas Legislative Council"); +}); + +void test("rejects a proposition from a different election date", () => { + const matched = matchTexasOfficialMeasure( + "Proposition 1", + { + stateAbbrev: "TX", + electionYear: 2025, + electionDate: "2025-05-03", + }, + current, + ); + assert.deepEqual(matched, {}); +}); diff --git a/packages/api/src/lib/measure-sources/texas-official.ts b/packages/api/src/lib/measure-sources/texas-official.ts new file mode 100644 index 00000000..7b190bda --- /dev/null +++ b/packages/api/src/lib/measure-sources/texas-official.ts @@ -0,0 +1,113 @@ +/** Current-cycle Texas SOS facts and TLC explanations for Google Civic measures. */ + +import type { TexasCurrentElectionData } from "../texas-election-data"; +import type { MeasureSourceData } from "./types"; +import { getTexasCurrentElectionData } from "../texas-election-data"; +import { titleSimilarity } from "../votesmart"; + +function propositionNumber(title: string): number | undefined { + const value = /\b(?:proposition|prop\.?)\s*(\d+)\b/i.exec(title)?.[1]; + return value ? Number.parseInt(value, 10) : undefined; +} + +function sameDate(a: string | undefined, b: string | undefined): boolean { + if (!a || !b) return true; + const left = new Date(a); + const right = new Date(b); + if (Number.isNaN(left.getTime()) || Number.isNaN(right.getTime())) { + return a.toLowerCase() === b.toLowerCase(); + } + return left.toISOString().slice(0, 10) === right.toISOString().slice(0, 10); +} + +/** + * Match by election date/year plus proposition number, falling back to a title + * similarity check only when Google omits the number. The returned SOS and TLC + * source objects stay separate so field citations cannot collapse them. + */ +export async function enrichFromTexasOfficial( + title: string, + context: { + stateAbbrev?: string; + electionYear: number; + electionDate?: string; + }, +): Promise<{ sos?: MeasureSourceData; tlc?: MeasureSourceData }> { + if (context.stateAbbrev?.toUpperCase() !== "TX") return {}; + const current = await getTexasCurrentElectionData().catch(() => null); + return matchTexasOfficialMeasure(title, context, current); +} + +/** Pure matcher used by the reader and fixture tests. */ +export function matchTexasOfficialMeasure( + title: string, + context: { + stateAbbrev?: string; + electionYear: number; + electionDate?: string; + }, + current: TexasCurrentElectionData | null, +): { sos?: MeasureSourceData; tlc?: MeasureSourceData } { + if (context.stateAbbrev?.toUpperCase() !== "TX") return {}; + const amendments = current?.constitutionalAmendments; + if ( + amendments?.cycleYear !== context.electionYear || + !sameDate(context.electionDate, amendments.electionDate) + ) { + return {}; + } + + const number = propositionNumber(title); + const measure = amendments.measures.find((candidate) => { + if (number !== undefined) return candidate.propositionNumber === number; + return titleSimilarity(title, candidate.ballotLanguage ?? "") >= 0.45; + }); + if (!measure) return {}; + + const tlc: MeasureSourceData = { + tier: "legislative_council", + sourceName: "Texas Legislative Council", + sourceUrl: measure.citations.summaryAnalysis?.sourceUrl, + official: true, + matchedTitle: measure.ballotLanguage, + officialSummary: measure.summaryAnalysis, + fiscalImpact: measure.fiscalImplications.join(" ") || undefined, + fullText: measure.ballotLanguage, + fullTextUrl: measure.citations.ballotLanguage?.sourceUrl, + proArguments: measure.supporterArguments.map((text) => ({ + text, + sourceName: "Texas Legislative Council", + sourceUrl: measure.citations.supporterArguments?.sourceUrl, + })), + conArguments: measure.opponentArguments.map((text) => ({ + text, + sourceName: "Texas Legislative Council", + sourceUrl: measure.citations.opponentArguments?.sourceUrl, + })), + }; + + const result = measure.result; + const sos: MeasureSourceData | undefined = result + ? { + tier: "state_sos", + sourceName: "Texas Secretary of State", + sourceUrl: result.citation.sourceUrl, + official: true, + matchedTitle: title, + result: { + status: result.status, + outcome: result.outcome, + totalVotes: result.totalVotes, + choices: result.choices.map((choice) => ({ + name: choice.name, + votes: choice.votes, + percent: choice.percent, + })), + sourceName: "Texas Secretary of State", + sourceUrl: result.citation.sourceUrl, + asOf: result.asOf, + }, + } + : undefined; + return { sos, tlc }; +} diff --git a/packages/api/src/lib/measure-sources/types.ts b/packages/api/src/lib/measure-sources/types.ts index d39683d0..6cb6acee 100644 --- a/packages/api/src/lib/measure-sources/types.ts +++ b/packages/api/src/lib/measure-sources/types.ts @@ -11,13 +11,14 @@ * Trust tiers, highest to lowest. The cross-validation engine prefers data * from a higher tier when multiple sources cover the same field. * - * County registrar / State SOS (official) > League of Women Voters + * County registrar / State SOS / legislative council (official) > League of Women Voters * (nonpartisan) > Ballotpedia (aggregator) > Wikipedia (encyclopedic) > * Vote Smart > Google Civic > AI grounded on fetched text (last resort). */ export type SourceTier = | "county_registrar" | "state_sos" + | "legislative_council" | "lwv" | "ballotpedia" | "wikipedia" @@ -28,6 +29,7 @@ export type SourceTier = export const SOURCE_TIER_RANK: Record = { county_registrar: 8, state_sos: 7, + legislative_council: 7, lwv: 6, ballotpedia: 5, wikipedia: 4, @@ -77,6 +79,18 @@ export interface MeasureSourceData { /** Single pro/con statement (Google Civic shape) when no list is available. */ proStatement?: string; conStatement?: string; + /** Official election result facts; explanatory sources leave this absent. */ + result?: MeasureResult; +} + +export interface MeasureResult { + status: "upcoming" | "reporting" | "complete" | "official"; + outcome?: "adopted" | "rejected"; + totalVotes: number; + choices: { name: string; votes: number; percent: number }[]; + asOf?: string; + sourceName: string; + sourceUrl: string; } /** @@ -97,6 +111,7 @@ export interface CanonicalMeasure { fullTextUrl?: string; proArguments: MeasureArgument[]; conArguments: MeasureArgument[]; + result?: MeasureResult; citations: MeasureCitation[]; /** Fields where sources disagreed, flagged for human review. */ discrepancies?: string[]; diff --git a/packages/api/src/lib/ncsbe-election-data.test.ts b/packages/api/src/lib/ncsbe-election-data.test.ts new file mode 100644 index 00000000..5f0ef86f --- /dev/null +++ b/packages/api/src/lib/ncsbe-election-data.test.ts @@ -0,0 +1,36 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { matchNcsbeName } from "./ncsbe-election-data"; + +void test("matches NCSBE contests exactly after party suffix normalization", () => { + assert.deepEqual( + matchNcsbeName("US SENATE (DEM)", ["US Senate", "NC Senate District 20"]), + { value: "US Senate", method: "exact", score: 1 }, + ); +}); + +void test("uses an unambiguous token fallback for Civic wording differences", () => { + assert.deepEqual( + matchNcsbeName("NC COURT APPEALS JUDGE SEAT 01", [ + "North Carolina Court of Appeals Judge Seat 01", + "North Carolina Supreme Court Associate Justice", + ]), + { + value: "North Carolina Court of Appeals Judge Seat 01", + method: "token", + score: 14 / 15, + }, + ); +}); + +void test("rejects ambiguous or weak fuzzy matches", () => { + assert.equal( + matchNcsbeName("County Board", [ + "County Board Seat 1", + "County Board Seat 2", + ]), + null, + ); + assert.equal(matchNcsbeName("Mayor", ["US Senate"]), null); +}); diff --git a/packages/api/src/lib/ncsbe-election-data.ts b/packages/api/src/lib/ncsbe-election-data.ts new file mode 100644 index 00000000..98a4d652 --- /dev/null +++ b/packages/api/src/lib/ncsbe-election-data.ts @@ -0,0 +1,240 @@ +/** Provider-neutral reader for normalized NCSBE public election records. */ +import { and, eq } from "@acme/db"; +import { db } from "@acme/db/client"; +import { + ElectionCandidate, + ElectionReferendum, + ElectionResult, + ElectionSource, +} from "@acme/db/schema"; + +export interface CivicContestReference { + title: string; + candidates?: string[]; +} + +export type ElectionMatch = { + value: string; + method: "exact" | "token"; + score: number; +} | null; + +/** + * Matching is exact after punctuation/case/party-suffix normalization. The + * documented fallback is token Dice similarity >= .82, accepted only when the + * best candidate is at least .08 ahead of the runner-up to avoid silent ties. + */ +export function matchNcsbeName( + source: string, + targets: readonly string[], +): ElectionMatch { + const normalizedSource = normalize(source); + const exact = targets.find( + (target) => normalize(target) === normalizedSource, + ); + if (exact) return { value: exact, method: "exact", score: 1 }; + const ranked = targets + .map((target) => ({ target, score: dice(tokens(source), tokens(target)) })) + .sort((a, b) => b.score - a.score || a.target.localeCompare(b.target)); + const best = ranked[0]; + const runnerUp = ranked[1]; + return best && + best.score >= 0.82 && + best.score - (runnerUp?.score ?? 0) >= 0.08 + ? { value: best.target, method: "token", score: best.score } + : null; +} + +function normalize(value: string): string { + return value + .toLowerCase() + .replace(/\((?:dem|rep|lib|gre|una|non)\)\s*$/i, "") + .replace(/\bnc\b/g, "north carolina") + .replace(/\b(?:the|office of)\b/g, " ") + .replace(/[^a-z0-9]+/g, " ") + .trim() + .replace(/\s+/g, " "); +} + +function tokens(value: string): Set { + return new Set(normalize(value).split(" ").filter(Boolean)); +} + +function dice(left: Set, right: Set): number { + if (left.size === 0 || right.size === 0) return 0; + let overlap = 0; + for (const token of left) if (right.has(token)) overlap++; + return (2 * overlap) / (left.size + right.size); +} + +function ensureCurrentCycle(electionDate: string, now: Date): void { + const year = Number(electionDate.slice(0, 4)); + if ( + !/^\d{4}-\d{2}-\d{2}$/.test(electionDate) || + year !== now.getUTCFullYear() + ) { + throw new Error("NCSBE reads are limited to the current election cycle"); + } +} + +function contestMatch(contest: string, refs: readonly CivicContestReference[]) { + return matchNcsbeName( + contest, + refs.map((ref) => ref.title), + ); +} + +export async function getCurrentNcElectionData(input: { + county: string; + electionDate: string; + civicContests?: readonly CivicContestReference[]; + includePrecincts?: boolean; + now?: Date; +}) { + ensureCurrentCycle(input.electionDate, input.now ?? new Date()); + const county = input.county + .trim() + .replace(/\s+county$/i, "") + .toUpperCase(); + const sourceFields = { + sourceUrl: ElectionSource.sourceUrl, + fetchedAt: ElectionSource.fetchedAt, + checksum: ElectionSource.checksum, + structureVersion: ElectionSource.structureVersion, + certificationStatus: ElectionSource.certificationStatus, + }; + + const [candidateRows, referendumRows, resultRows] = await Promise.all([ + db + .select({ + electionDate: ElectionCandidate.electionDate, + county: ElectionCandidate.county, + contest: ElectionCandidate.contest, + name: ElectionCandidate.name, + party: ElectionCandidate.party, + voteFor: ElectionCandidate.voteFor, + termYears: ElectionCandidate.termYears, + hasPrimary: ElectionCandidate.hasPrimary, + isPartisan: ElectionCandidate.isPartisan, + ...sourceFields, + }) + .from(ElectionCandidate) + .innerJoin( + ElectionSource, + eq(ElectionCandidate.sourceId, ElectionSource.id), + ) + .where( + and( + eq(ElectionCandidate.electionDate, input.electionDate), + eq(ElectionCandidate.county, county), + ), + ), + db + .select({ + electionDate: ElectionReferendum.electionDate, + county: ElectionReferendum.county, + contest: ElectionReferendum.contest, + choice: ElectionReferendum.choice, + description: ElectionReferendum.description, + ...sourceFields, + }) + .from(ElectionReferendum) + .innerJoin( + ElectionSource, + eq(ElectionReferendum.sourceId, ElectionSource.id), + ) + .where( + and( + eq(ElectionReferendum.electionDate, input.electionDate), + eq(ElectionReferendum.county, county), + ), + ), + db + .select({ + electionDate: ElectionResult.electionDate, + county: ElectionResult.county, + precinct: ElectionResult.precinct, + contestId: ElectionResult.contestId, + contestType: ElectionResult.contestType, + contest: ElectionResult.contest, + choice: ElectionResult.choice, + party: ElectionResult.party, + voteFor: ElectionResult.voteFor, + electionDayVotes: ElectionResult.electionDayVotes, + earlyVotingVotes: ElectionResult.earlyVotingVotes, + absenteeMailVotes: ElectionResult.absenteeMailVotes, + provisionalVotes: ElectionResult.provisionalVotes, + totalVotes: ElectionResult.totalVotes, + realPrecinct: ElectionResult.realPrecinct, + ...sourceFields, + }) + .from(ElectionResult) + .innerJoin(ElectionSource, eq(ElectionResult.sourceId, ElectionSource.id)) + .where( + and( + eq(ElectionResult.electionDate, input.electionDate), + eq(ElectionResult.county, county), + ), + ), + ]); + + const refs = input.civicContests ?? []; + const withContestMatch = ( + rows: T[], + ): (T & { civicMatch: ElectionMatch })[] => { + const matched: (T & { civicMatch: ElectionMatch })[] = []; + for (const row of rows) { + if (refs.length === 0) { + matched.push({ ...row, civicMatch: null }); + continue; + } + const civicMatch = contestMatch(row.contest, refs); + if (civicMatch) matched.push({ ...row, civicMatch }); + } + return matched; + }; + const candidates = withContestMatch(candidateRows).map((row) => { + const ref = refs.find( + (candidate) => candidate.title === row.civicMatch?.value, + ); + return { + ...row, + civicCandidateMatch: ref?.candidates?.length + ? matchNcsbeName(row.name, ref.candidates) + : null, + }; + }); + const referenda = withContestMatch(referendumRows); + const precinctResults = withContestMatch(resultRows); + const totals = new Map< + string, + Omit<(typeof precinctResults)[number], "precinct" | "realPrecinct"> + >(); + for (const row of precinctResults) { + const key = `${row.contest}\u0000${row.choice}\u0000${row.party ?? ""}`; + const existing = totals.get(key); + if (existing) { + existing.electionDayVotes += row.electionDayVotes; + existing.earlyVotingVotes += row.earlyVotingVotes; + existing.absenteeMailVotes += row.absenteeMailVotes; + existing.provisionalVotes += row.provisionalVotes; + existing.totalVotes += row.totalVotes; + } else { + const { + precinct: _precinct, + realPrecinct: _realPrecinct, + ...total + } = row; + totals.set(key, total); + } + } + return { + provider: "ncsbe" as const, + electionDate: input.electionDate, + county, + candidates, + referenda, + results: [...totals.values()], + precinctResults: input.includePrecincts ? precinctResults : undefined, + }; +} diff --git a/packages/api/src/lib/texas-election-data.test.ts b/packages/api/src/lib/texas-election-data.test.ts new file mode 100644 index 00000000..e47b22fa --- /dev/null +++ b/packages/api/src/lib/texas-election-data.test.ts @@ -0,0 +1,113 @@ +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import test from "node:test"; + +import { + parseTexasSosDiscovery, + parseTexasSosElection, + parseTexasTlcAnalysis, + TEXAS_SOS_PROVIDER, + TEXAS_TLC_PROVIDER, +} from "./texas-election-data"; + +function fixture(name: string): T { + return JSON.parse( + readFileSync(new URL(`./__fixtures__/${name}`, import.meta.url), "utf8"), + ) as T; +} + +void test("SOS discovery selects the current cycle without hard-coded year URLs", () => { + const discovery = parseTexasSosDiscovery(fixture("texas-sos-constants.json")); + assert.equal(discovery.cycleYear, 2026); + assert.deepEqual( + discovery.elections.map((election) => election.id), + [53813, 53815], + ); + assert.equal(discovery.electionsByYear[2025]?.[0]?.id, 51031); +}); + +void test("SOS parser normalizes current candidate contests and official status", () => { + const discovery = parseTexasSosDiscovery(fixture("texas-sos-constants.json")); + const definition = discovery.elections.find( + (election) => election.id === 53813, + ); + assert.ok(definition); + const election = parseTexasSosElection( + fixture("texas-sos-election.json"), + definition, + undefined, + new Date("2026-07-21T00:00:00Z"), + ); + assert.equal(election.status, "official"); + assert.equal(election.reporting.percentReporting, 100); + const contest = election.contests[0]; + assert.ok(contest); + const choice = contest.choices[0]; + assert.ok(choice); + assert.equal(contest.title, "GOVERNOR"); + assert.equal(choice.name, "GREG ABBOTT"); + assert.equal(choice.incumbent, true); + assert.equal(contest.citation.provider, TEXAS_SOS_PROVIDER); +}); + +void test("SOS amendment results include outcome, county totals, and turnout", () => { + const discovery = parseTexasSosDiscovery(fixture("texas-sos-constants.json")); + const definition = discovery.electionsByYear[2025]?.[0]; + assert.ok(definition); + const election = parseTexasSosElection( + fixture("texas-sos-amendment.json"), + definition, + fixture("texas-sos-county.json"), + new Date("2026-07-21T00:00:00Z"), + ); + const contest = election.contests[0]; + assert.ok(contest); + assert.equal(election.status, "complete"); + assert.equal(contest.type, "referendum"); + assert.equal(contest.propositionNumber, 1); + assert.equal(contest.outcome, "adopted"); + const county = contest.counties[0]; + assert.ok(county); + const countyChoice = county.choices[0]; + assert.ok(countyChoice); + assert.equal(county.county, "ANDERSON"); + assert.equal(countyChoice.votes, 3361); + assert.ok(election.turnout); + assert.equal(election.turnout.registeredVoters, 80544); + assert.equal(election.turnout.ballotsCast, 4739); +}); + +void test("TLC 2025 layout preserves page citations and separate explanation tier", () => { + const parsed = parseTexasTlcAnalysis( + fixture("texas-tlc-2025.json"), + "https://tlc.texas.gov/docs/amendments/analyses25.pdf", + ); + assert.equal(parsed.cycleYear, 2025); + assert.equal(parsed.electionDate, "NOVEMBER 4, 2025"); + assert.equal(parsed.measures.length, 2); + const first = parsed.measures[0]; + assert.ok(first); + assert.equal(first.resolution, "SJR 59"); + assert.match(first.ballotLanguage ?? "", /permanent technical institution/i); + assert.equal(first.supporterArguments.length, 2); + assert.equal(first.opponentArguments.length, 1); + assert.match(first.fiscalImplications[0] ?? "", /\$850 million/); + const summaryCitation = first.citations.summaryAnalysis; + assert.ok(summaryCitation); + assert.equal(summaryCitation.provider, TEXAS_TLC_PROVIDER); + assert.equal(summaryCitation.page, 10); + assert.match(summaryCitation.sourceUrl, /#page=10$/); +}); + +void test("TLC 2023 alternate layout fails soft when a section is absent", () => { + const parsed = parseTexasTlcAnalysis( + fixture("texas-tlc-2023.json"), + "https://tlc.texas.gov/docs/amendments/analyses23.pdf", + ); + const measure = parsed.measures[0]; + assert.ok(measure); + assert.equal(measure.propositionNumber, 1); + assert.match(measure.background ?? "", /Existing statutes/); + assert.deepEqual(measure.opponentArguments, []); + assert.ok(measure.diagnostics.includes("missing opponent comments")); +}); diff --git a/packages/api/src/lib/texas-election-data.ts b/packages/api/src/lib/texas-election-data.ts new file mode 100644 index 00000000..adb1865d --- /dev/null +++ b/packages/api/src/lib/texas-election-data.ts @@ -0,0 +1,808 @@ +/** + * Deterministic parsers and reader types for the Texas SOS/TLC current-cycle + * election handoff. SOS result facts and TLC explanatory text deliberately use + * distinct records and citations all the way to the API response. + */ + +import { and, desc, eq } from "@acme/db"; +import { db } from "@acme/db/client"; +import { ElectionSourceSnapshot } from "@acme/db/schema"; + +export const TEXAS_SOS_PROVIDER = "texas-sos"; +export const TEXAS_TLC_PROVIDER = "texas-tlc"; +export const TEXAS_CURRENT_SCOPE = "current"; +export const TEXAS_RESULTS_URL = + "https://goelect.txelections.civixapps.com/ivis-enr-ui/races"; +export const TEXAS_TLC_PUBLICATIONS_URL = "https://tlc.texas.gov/publications"; + +export interface SourceCitation { + sourceName: string; + sourceUrl: string; + official: true; + provider: typeof TEXAS_SOS_PROVIDER | typeof TEXAS_TLC_PROVIDER; + page?: number; +} + +export interface TexasElectionDefinition { + id: number; + name: string; + year: number; + type: string; + category?: string; + official: boolean; +} + +export interface TexasSosDiscovery { + cycleYear: number; + elections: TexasElectionDefinition[]; + electionsByYear: Record; +} + +export interface TexasResultChoice { + id?: number; + name: string; + party?: string; + incumbent: boolean; + votes: number; + earlyVotes?: number; + percent: number; + winner: boolean; +} + +export interface TexasCountyResult { + countyId: number; + county: string; + registeredVoters?: number; + ballotsCast?: number; + turnoutPercent?: number; + reportingPercent?: number; + choices: TexasResultChoice[]; +} + +export interface TexasElectionContest { + id: string; + sourceContestId: number; + type: "candidate" | "referendum"; + title: string; + officeType?: string; + propositionNumber?: number; + resolution?: string; + totalVotes: number; + choices: TexasResultChoice[]; + counties: TexasCountyResult[]; + outcome?: "adopted" | "rejected"; + citation: SourceCitation; +} + +export interface TexasElectionReporting { + countiesReporting?: number; + countiesTotal?: number; + precinctsReporting?: number; + precinctsTotal?: number; + pollingPlacesReporting?: number; + pollingPlacesTotal?: number; + percentReporting?: number; +} + +export interface TexasElectionTurnout { + registeredVoters?: number; + ballotsCast?: number; + turnoutPercent?: number; +} + +export interface TexasSosElection { + id: number; + name: string; + date?: string; + year: number; + electionType: string; + status: "upcoming" | "reporting" | "complete" | "official"; + official: boolean; + lastUpdated?: string; + sourceVersion: string; + reporting: TexasElectionReporting; + turnout?: TexasElectionTurnout; + contests: TexasElectionContest[]; + citation: SourceCitation; +} + +export interface TexasSosSnapshotData { + cycleYear: number; + elections: TexasSosElection[]; +} + +export interface TlcTextPage { + page: number; + text: string; +} + +export interface TexasTlcMeasure { + propositionNumber: number; + resolution?: string; + ballotLanguage?: string; + summaryAnalysis?: string; + background?: string; + supporterArguments: string[]; + opponentArguments: string[]; + fiscalImplications: string[]; + pageStart: number; + pageEnd: number; + citations: { + ballotLanguage?: SourceCitation; + summaryAnalysis?: SourceCitation; + background?: SourceCitation; + supporterArguments?: SourceCitation; + opponentArguments?: SourceCitation; + fiscalImplications?: SourceCitation; + }; + diagnostics: string[]; +} + +export interface TexasTlcSnapshotData { + cycleYear: number; + electionDate?: string; + publicationTitle?: string; + sourceUrl: string; + measures: TexasTlcMeasure[]; +} + +export interface TexasCurrentElectionData { + jurisdiction: "TX"; + cycleYear: number; + elections: TexasSosElection[]; + constitutionalAmendments?: { + cycleYear: number; + electionDate?: string; + publicationTitle?: string; + measures: (TexasTlcMeasure & { + result?: Pick< + TexasElectionContest, + "outcome" | "totalVotes" | "choices" | "citation" + > & { + status: TexasSosElection["status"]; + asOf?: string; + }; + })[]; + citation: SourceCitation; + }; + diagnostics: string[]; + fetchedAt: string; +} + +type JsonObject = Record; + +function object(value: unknown): JsonObject | null { + return value !== null && typeof value === "object" && !Array.isArray(value) + ? (value as JsonObject) + : null; +} + +function stringValue(value: unknown): string | undefined { + return typeof value === "string" && value.trim() ? value.trim() : undefined; +} + +function numberValue(value: unknown): number | undefined { + const parsed = typeof value === "number" ? value : Number(value); + return Number.isFinite(parsed) ? parsed : undefined; +} + +function decodeJson(value: unknown): unknown { + if (typeof value !== "string" || !value) return null; + try { + return JSON.parse(Buffer.from(value, "base64").toString("utf8")) as unknown; + } catch { + return null; + } +} + +function unwrapUpload(value: unknown): unknown { + const envelope = object(value); + return envelope && "upload" in envelope ? decodeJson(envelope.upload) : value; +} + +/** Parse the SOS discovery envelope and return only structured definitions. */ +export function parseTexasSosDiscovery(raw: unknown): TexasSosDiscovery { + const decoded = object(unwrapUpload(raw)); + const years = object(decoded?.electionInfo); + if (!years) + throw new Error("Texas SOS discovery payload has no electionInfo"); + + const electionsByYear: Record = {}; + for (const [yearKey, groupsValue] of Object.entries(years)) { + const year = Number.parseInt(yearKey, 10); + const groups = object(groupsValue); + if (!Number.isInteger(year) || !groups) continue; + const definitions: TexasElectionDefinition[] = []; + for (const group of Object.values(groups)) { + const entries = object(group); + if (!entries) continue; + for (const candidate of Object.values(entries)) { + const item = object(candidate); + const id = numberValue(item?.ID); + const name = stringValue(item?.N); + if (id === undefined || !name) continue; + definitions.push({ + id, + name: name.replace(/\s+/g, " "), + year, + type: stringValue(item?.ET) ?? "unknown", + category: stringValue(item?.EC), + official: item?.O === "Y", + }); + } + } + electionsByYear[year] = definitions.sort((a, b) => a.id - b.id); + } + const cycleYear = Math.max(...Object.keys(electionsByYear).map(Number)); + if (!Number.isFinite(cycleYear)) { + throw new Error("Texas SOS discovery payload contains no election years"); + } + return { + cycleYear, + elections: electionsByYear[cycleYear] ?? [], + electionsByYear, + }; +} + +function cleanCandidateName(name: string): { + name: string; + incumbent: boolean; +} { + return { + name: name.replace(/\s*\(I\)\s*$/i, "").trim(), + incumbent: /\(I\)\s*$/i.test(name), + }; +} + +function parseChoice(value: unknown): TexasResultChoice | null { + const item = object(value); + const rawName = stringValue(item?.N); + if (!item || !rawName) return null; + const cleaned = cleanCandidateName(rawName); + return { + id: numberValue(item.ID ?? item.id), + name: cleaned.name, + party: stringValue(item.P), + incumbent: cleaned.incumbent, + votes: numberValue(item.V) ?? 0, + earlyVotes: numberValue(item.EV), + percent: numberValue(item.PE) ?? 0, + winner: false, + }; +} + +function propositionIdentity(title: string): { + number?: number; + resolution?: string; +} { + const number = /\bproposition\s+(\d+)\b/i.exec(title)?.[1]; + const resolution = /\(([SH]\.?\s*J\.?\s*R\.?)\s*(\d+)\)/i.exec(title); + return { + number: number ? Number.parseInt(number, 10) : undefined, + resolution: resolution + ? `${(resolution[1] ?? "").replace(/\s|\./g, "").toUpperCase()} ${resolution[2]}` + : undefined, + }; +} + +interface RaceSeed { + id: number; + title: string; + officeType?: string; + totalVotes: number; + choices: TexasResultChoice[]; +} + +function raceSeeds(raw: JsonObject): RaceSeed[] { + const groups: unknown[] = []; + const raceRoot = object(decodeJson(raw.Race)); + const officeTypes: unknown[] = Array.isArray(raceRoot?.OfficeTypes) + ? (raceRoot.OfficeTypes as unknown[]) + : []; + groups.push(...officeTypes); + for (const key of ["StateWide", "StateWideQ", "Districted", "Federal"]) { + const section = object(decodeJson(raw[key])); + if (section) groups.push(section); + } + + const byId = new Map(); + for (const groupValue of groups) { + const group = object(groupValue); + if (!group) continue; + const races = Array.isArray(group.Races) ? group.Races : []; + for (const raceValue of races) { + const race = object(raceValue); + if (!race) continue; + const id = numberValue(race.id ?? race.OID); + const title = stringValue(race.N); + if (id === undefined || !title || byId.has(id)) continue; + const choices = ( + Array.isArray(race.Candidates) + ? race.Candidates + : Object.values(object(race.C) ?? {}) + ) + .map(parseChoice) + .filter((choice): choice is TexasResultChoice => choice !== null) + .sort((a, b) => b.votes - a.votes); + const maxVotes = choices[0]?.votes ?? 0; + for (const choice of choices) + choice.winner = maxVotes > 0 && choice.votes === maxVotes; + byId.set(id, { + id, + title: title.replace(/\s+/g, " ").trim(), + officeType: stringValue(group.OfficeType ?? race.OT), + totalVotes: + numberValue(race.T) ?? + choices.reduce((sum, choice) => sum + choice.votes, 0), + choices, + }); + } + } + return [...byId.values()]; +} + +function reportingPercent( + reporting: TexasElectionReporting, +): number | undefined { + const pairs: [number | undefined, number | undefined][] = [ + [reporting.pollingPlacesReporting, reporting.pollingPlacesTotal], + [reporting.countiesReporting, reporting.countiesTotal], + [reporting.precinctsReporting, reporting.precinctsTotal], + ]; + for (const [done, total] of pairs) { + if (done !== undefined && total !== undefined && total > 0) { + return Math.min(100, Math.round((done / total) * 10_000) / 100); + } + } + return undefined; +} + +function isoDate(mmddyyyy: string | undefined): string | undefined { + if (!mmddyyyy || !/^\d{8}$/.test(mmddyyyy)) return undefined; + return `${mmddyyyy.slice(4)}-${mmddyyyy.slice(0, 2)}-${mmddyyyy.slice(2, 4)}`; +} + +function parseCountyResults( + raw: unknown, +): Map< + number, + { totals: TexasElectionTurnout; results: Map } +> { + const decoded = object(unwrapUpload(raw)); + const result = new Map< + number, + { totals: TexasElectionTurnout; results: Map } + >(); + if (!decoded) return result; + + for (const [countyIdKey, countyValue] of Object.entries(decoded)) { + const county = object(countyValue); + const countyId = Number.parseInt(countyIdKey, 10); + const name = stringValue(county?.N); + if (!county || !Number.isInteger(countyId) || !name) continue; + const summary = object(county.Summary); + const registeredVoters = numberValue(county.TV ?? summary?.RV); + const ballotsCast = numberValue(summary?.VC); + const turnoutPercent = + numberValue(summary?.VT) ?? + (registeredVoters && ballotsCast !== undefined + ? Math.round((ballotsCast / registeredVoters) * 10_000) / 100 + : undefined); + const byRace = new Map(); + const races = object(county.Races) ?? {}; + for (const raceValue of Object.values(races)) { + const race = object(raceValue); + const raceId = numberValue(race?.OID); + if (!race || raceId === undefined) continue; + const choices = Object.values(object(race.C) ?? {}) + .map(parseChoice) + .filter((choice): choice is TexasResultChoice => choice !== null) + .sort((a, b) => b.votes - a.votes); + const maxVotes = choices[0]?.votes ?? 0; + for (const choice of choices) + choice.winner = maxVotes > 0 && choice.votes === maxVotes; + const precinctsReporting = numberValue(race.PR); + const precinctsTotal = numberValue(race.TP); + byRace.set(raceId, [ + { + countyId, + county: name, + registeredVoters, + ballotsCast, + turnoutPercent, + reportingPercent: + precinctsReporting !== undefined && precinctsTotal + ? Math.min( + 100, + Math.round((precinctsReporting / precinctsTotal) * 10_000) / + 100, + ) + : undefined, + choices, + }, + ]); + } + result.set(countyId, { + totals: { registeredVoters, ballotsCast, turnoutPercent }, + results: byRace, + }); + } + return result; +} + +/** Normalize one SOS election payload, optionally including county totals. */ +export function parseTexasSosElection( + raw: unknown, + definition: TexasElectionDefinition, + countyRaw?: unknown, + today = new Date(), +): TexasSosElection { + const root = object(raw); + if (!root) + throw new Error(`Texas SOS election ${definition.id} is not an object`); + const home = object(decodeJson(root.Home)) ?? {}; + const reporting: TexasElectionReporting = { + countiesReporting: numberValue(object(home.CountiesReporting)?.CR), + countiesTotal: numberValue(object(home.CountiesReporting)?.CT), + precinctsReporting: numberValue(object(home.PrecinctsReporting)?.PR), + precinctsTotal: numberValue(object(home.PrecinctsReporting)?.PT), + pollingPlacesReporting: numberValue(object(home.PollingReporting)?.PLR), + pollingPlacesTotal: numberValue(object(home.PollingReporting)?.PLT), + }; + reporting.percentReporting = reportingPercent(reporting); + const date = isoDate(stringValue(home.ElecDate)); + const counties = parseCountyResults(countyRaw); + const countyRows = [...counties.values()]; + const registeredVoters = countyRows.reduce( + (sum, row) => sum + (row.totals.registeredVoters ?? 0), + 0, + ); + const ballotsCast = countyRows.reduce( + (sum, row) => sum + (row.totals.ballotsCast ?? 0), + 0, + ); + const citation: SourceCitation = { + sourceName: "Texas Secretary of State", + sourceUrl: TEXAS_RESULTS_URL, + official: true, + provider: TEXAS_SOS_PROVIDER, + }; + const contests = raceSeeds(root).map((race): TexasElectionContest => { + const identity = propositionIdentity(race.title); + const type = identity.number === undefined ? "candidate" : "referendum"; + const contestCounties: TexasCountyResult[] = []; + for (const county of counties.values()) { + contestCounties.push(...(county.results.get(race.id) ?? [])); + } + const forChoice = race.choices.find((choice) => /^for$/i.test(choice.name)); + const againstChoice = race.choices.find((choice) => + /^against$/i.test(choice.name), + ); + return { + id: `tx-sos:${definition.id}:${race.id}`, + sourceContestId: race.id, + type, + title: race.title, + officeType: race.officeType, + propositionNumber: identity.number, + resolution: identity.resolution, + totalVotes: race.totalVotes, + choices: race.choices, + counties: contestCounties, + outcome: + forChoice && againstChoice && race.totalVotes > 0 + ? forChoice.votes > againstChoice.votes + ? "adopted" + : "rejected" + : undefined, + citation, + }; + }); + const totalVotes = contests.reduce( + (sum, contest) => sum + contest.totalVotes, + 0, + ); + const electionDate = date ? new Date(`${date}T23:59:59Z`) : undefined; + const complete = (reporting.percentReporting ?? 0) >= 100; + const status = definition.official + ? "official" + : electionDate && electionDate > today && totalVotes === 0 + ? "upcoming" + : complete + ? "complete" + : totalVotes > 0 + ? "reporting" + : "upcoming"; + return { + id: definition.id, + name: definition.name, + date, + year: definition.year, + electionType: definition.type, + status, + official: definition.official, + lastUpdated: stringValue(home.LastUpdatedTime), + sourceVersion: stringValue(root.Version) ?? `election/${definition.id}`, + reporting, + turnout: + registeredVoters > 0 + ? { + registeredVoters, + ballotsCast, + turnoutPercent: + Math.round((ballotsCast / registeredVoters) * 10_000) / 100, + } + : undefined, + contests, + citation, + }; +} + +function compactText(value: string): string { + return value + .replace(/\u00ad/g, "") + .replace(/[ \t]+/g, " ") + .replace(/\s*\n\s*/g, " ") + .replace(/\s+/g, " ") + .trim(); +} + +function section( + text: string, + start: RegExp, + ends: RegExp[], +): string | undefined { + const match = start.exec(text); + if (!match) return undefined; + const tail = text.slice(match.index + match[0].length); + let end = tail.length; + for (const pattern of ends) { + const next = pattern.exec(tail); + if (next && next.index < end) end = next.index; + } + const value = compactText(tail.slice(0, end)); + return value || undefined; +} + +function argumentList(text: string | undefined): string[] { + if (!text) return []; + const pieces = text.includes("•") ? text.split("•") : text.split(/\n\s*[-–]/); + return pieces.map(compactText).filter((item) => item.length >= 15); +} + +function fiscalSentences(...texts: (string | undefined)[]): string[] { + const seen = new Set(); + const output: string[] = []; + for (const text of texts) { + for (const sentence of (text ?? "").split(/(?<=[.!?])\s+/)) { + const value = compactText(sentence); + if ( + value.length >= 30 && + /\b(?:fiscal|tax|revenue|fund(?:ing)?|appropriat|cost|dollars?|\$)\b/i.test( + value, + ) && + !seen.has(value.toLowerCase()) + ) { + seen.add(value.toLowerCase()); + output.push(value); + } + if (output.length >= 8) return output; + } + } + return output; +} + +function tlcCitation(sourceUrl: string, page: number): SourceCitation { + return { + sourceName: "Texas Legislative Council", + sourceUrl: `${sourceUrl}#page=${page}`, + official: true, + provider: TEXAS_TLC_PROVIDER, + page, + }; +} + +/** Parse text-layer pages from either the 2023 or 2025 TLC report layout. */ +export function parseTexasTlcAnalysis( + pages: TlcTextPage[], + sourceUrl: string, +): TexasTlcSnapshotData { + const cover = compactText( + pages + .slice(0, 8) + .map((page) => page.text) + .join(" "), + ); + const year = Number.parseInt( + /(?:19|20)\d{2}/.exec(cover)?.[0] ?? String(new Date().getFullYear()), + 10, + ); + const electionDate = + /\b(November|May|September)\s+\d{1,2},\s+(?:19|20)\d{2}\b/i.exec( + cover, + )?.[0]; + const starts: { index: number; propositionNumber: number }[] = []; + for (let i = 0; i < pages.length; i++) { + const page = pages[i]; + if (!page) continue; + const text = page.text; + // The ballot language between the heading and SUMMARY ANALYSIS ranges from + // one short line to several wrapped lines (notably Props. 7, 12, 14, 17 in + // the 2025 layout). Keep the bound finite so table-of-contents pages cannot + // accidentally bridge into an unrelated heading. + const match = + /\bProposition\s+(\d+)\b[\s\S]{0,1200}\bSUMMARY\s+ANALYSIS\b/i.exec(text); + if (match?.[1]) + starts.push({ index: i, propositionNumber: Number(match[1]) }); + } + + const measures = starts.map((start, position): TexasTlcMeasure => { + const endIndex = (starts[position + 1]?.index ?? pages.length) - 1; + const text = pages + .slice(start.index, endIndex + 1) + .map((page) => page.text) + .join("\n"); + const heading = + /\bProposition\s+(\d+)\s*\(([SH]\.?\s*J\.?\s*R\.?)\s*(\d+)\)\s*/i.exec( + text, + ); + const afterHeading = heading + ? text.slice(heading.index + heading[0].length) + : text; + const ballotLanguage = compactText( + afterHeading.split(/\bSUMMARY\s+ANALYSIS\b/i)[0] ?? "", + ); + const summaryAnalysis = section(text, /\bSUMMARY\s+ANALYSIS\b/i, [ + /\bBACKGROUND(?:\s+AND\s+DETAILED\s+ANALYSIS)?\b/i, + /\bSUMMARY\s+OF\s+COMMENTS\b/i, + ]); + const background = section( + text, + /\bBACKGROUND(?:\s+AND\s+DETAILED\s+ANALYSIS)?\b/i, + [/\bSUMMARY\s+OF\s+COMMENTS\b/i, /\bCOMMENTS\s+BY\s+SUPPORTERS\b/i], + ); + const supporters = section(text, /\bComments\s+by\s+Supporters\s*:?/i, [ + /\bComments\s+by\s+Opponents\s*:?/i, + /\bText\s+of\s+[SH]\.?\s*J\.?\s*R\.?/i, + ]); + const opponents = section(text, /\bComments\s+by\s+Opponents\s*:?/i, [ + /\bText\s+of\s+[SH]\.?\s*J\.?\s*R\.?/i, + ]); + const startPage = pages[start.index]; + const endPage = pages[Math.max(start.index, endIndex)]; + if (!startPage || !endPage) { + throw new Error("TLC proposition page bounds are invalid"); + } + const pageStart = startPage.page; + const pageEnd = endPage.page; + const citation = tlcCitation(sourceUrl, pageStart); + const supporterArguments = argumentList(supporters); + const opponentArguments = argumentList(opponents); + const fiscalImplications = fiscalSentences(summaryAnalysis, background); + const diagnostics: string[] = []; + if (!ballotLanguage) diagnostics.push("missing ballot language"); + if (!summaryAnalysis) diagnostics.push("missing summary analysis"); + if (!background) diagnostics.push("missing background analysis"); + if (!supporterArguments.length) + diagnostics.push("missing supporter comments"); + if (!opponentArguments.length) + diagnostics.push("missing opponent comments"); + return { + propositionNumber: start.propositionNumber, + resolution: heading + ? `${(heading[2] ?? "").replace(/\s|\./g, "").toUpperCase()} ${heading[3]}` + : undefined, + ballotLanguage: ballotLanguage || undefined, + summaryAnalysis, + background, + supporterArguments, + opponentArguments, + fiscalImplications, + pageStart, + pageEnd, + citations: { + ballotLanguage: ballotLanguage ? citation : undefined, + summaryAnalysis: summaryAnalysis ? citation : undefined, + background: background ? citation : undefined, + supporterArguments: supporterArguments.length ? citation : undefined, + opponentArguments: opponentArguments.length ? citation : undefined, + fiscalImplications: fiscalImplications.length ? citation : undefined, + }, + diagnostics, + }; + }); + + return { + cycleYear: year, + electionDate, + publicationTitle: `Analyses of Proposed Constitutional Amendments (${year})`, + sourceUrl, + measures, + }; +} + +function asSosData( + value: Record, +): TexasSosSnapshotData | null { + return typeof value.cycleYear === "number" && Array.isArray(value.elections) + ? (value as unknown as TexasSosSnapshotData) + : null; +} + +function asTlcData( + value: Record, +): TexasTlcSnapshotData | null { + return typeof value.cycleYear === "number" && Array.isArray(value.measures) + ? (value as unknown as TexasTlcSnapshotData) + : null; +} + +/** Read and join only the current SOS cycle plus the latest TLC amendment cycle. */ +export async function getTexasCurrentElectionData(): Promise { + const rows = await db + .select() + .from(ElectionSourceSnapshot) + .where( + and( + eq(ElectionSourceSnapshot.jurisdiction, "TX"), + eq(ElectionSourceSnapshot.scope, TEXAS_CURRENT_SCOPE), + ), + ) + .orderBy(desc(ElectionSourceSnapshot.cycleYear)); + const sosRows = rows.filter((row) => row.provider === TEXAS_SOS_PROVIDER); + const tlcRow = rows.find((row) => row.provider === TEXAS_TLC_PROVIDER); + const tlc = tlcRow ? asTlcData(tlcRow.data) : null; + const currentSos = sosRows.map((row) => asSosData(row.data)).find(Boolean); + if (!currentSos && !tlc) return null; + const amendmentSos = tlc + ? sosRows + .map((row) => asSosData(row.data)) + .find((data) => data?.cycleYear === tlc.cycleYear) + : null; + const amendmentElection = amendmentSos?.elections.find((election) => + election.contests.some((contest) => contest.type === "referendum"), + ); + const byProposition = new Map( + amendmentElection?.contests.flatMap((contest) => + contest.propositionNumber === undefined + ? [] + : ([[contest.propositionNumber, contest]] as const), + ) ?? [], + ); + const fetchedAt = rows.reduce( + (latest, row) => (row.fetchedAt > latest ? row.fetchedAt : latest), + new Date(0), + ); + return { + jurisdiction: "TX", + cycleYear: currentSos?.cycleYear ?? tlc?.cycleYear ?? 0, + elections: currentSos?.elections ?? [], + constitutionalAmendments: tlc + ? { + cycleYear: tlc.cycleYear, + electionDate: tlc.electionDate, + publicationTitle: tlc.publicationTitle, + measures: tlc.measures.map((measure) => { + const contest = byProposition.get(measure.propositionNumber); + return { + ...measure, + result: + contest && amendmentElection + ? { + outcome: contest.outcome, + totalVotes: contest.totalVotes, + choices: contest.choices, + citation: contest.citation, + status: amendmentElection.status, + asOf: amendmentElection.lastUpdated, + } + : undefined, + }; + }), + citation: tlcCitation(tlc.sourceUrl, 1), + } + : undefined, + diagnostics: rows.flatMap((row) => row.diagnostics), + fetchedAt: fetchedAt.toISOString(), + }; +} diff --git a/packages/api/src/lib/votesmart.ts b/packages/api/src/lib/votesmart.ts index 3e40ac88..0528ea80 100644 --- a/packages/api/src/lib/votesmart.ts +++ b/packages/api/src/lib/votesmart.ts @@ -221,6 +221,7 @@ export async function enrichFromVoteSmart( referendumTitle: string, stateAbbrev: string, electionYear: number, + electionDate?: string, ): Promise { if (!getApiKey()) return null; @@ -238,6 +239,18 @@ export async function enrichFromVoteSmart( let best: VoteSmartMeasure | null = null; let bestScore = 0; for (const m of measures) { + if (electionDate && m.electionDate) { + const requested = new Date(electionDate); + const candidate = new Date(m.electionDate); + if ( + !Number.isNaN(requested.getTime()) && + !Number.isNaN(candidate.getTime()) && + requested.toISOString().slice(0, 10) !== + candidate.toISOString().slice(0, 10) + ) { + continue; + } + } const score = titleSimilarity(referendumTitle, m.title); if (score > bestScore) { bestScore = score; diff --git a/packages/api/src/root.ts b/packages/api/src/root.ts index ce0f4159..c426f42d 100644 --- a/packages/api/src/root.ts +++ b/packages/api/src/root.ts @@ -3,6 +3,7 @@ import { civicRouter } from "./router/civic"; import { contentRouter } from "./router/content"; import { feedbackRouter } from "./router/feedback"; import { legistarRouter } from "./router/legistar"; +import { localGovernmentRouter } from "./router/local-government"; import { openStatesRouter } from "./router/open-states"; import { placesRouter } from "./router/places"; import { postRouter } from "./router/post"; @@ -14,6 +15,7 @@ export const appRouter = createTRPCRouter({ auth: authRouter, civic: civicRouter, legistar: legistarRouter, + localGovernment: localGovernmentRouter, openStates: openStatesRouter, places: placesRouter, post: postRouter, diff --git a/packages/api/src/router/civic.ts b/packages/api/src/router/civic.ts index a921b1c6..39ffa34f 100644 --- a/packages/api/src/router/civic.ts +++ b/packages/api/src/router/civic.ts @@ -10,6 +10,8 @@ import { getVoterInfo, } from "../lib/civic"; import { getElectedOfficials } from "../lib/elected-officials"; +import { getCurrentNcElectionData } from "../lib/ncsbe-election-data"; +import { getTexasCurrentElectionData } from "../lib/texas-election-data"; import { publicProcedure } from "../trpc"; const STATEWIDE_OFFICE = z.enum( @@ -42,6 +44,25 @@ export const civicRouter = { } }), + /** + * Current-cycle Texas statewide election data from the persisted SOS/TLC + * handoff. This intentionally exposes no historical browsing parameter. + */ + getTexasCurrentElection: publicProcedure.query(async () => { + try { + return await getTexasCurrentElectionData(); + } catch (error) { + throw new TRPCError({ + code: "INTERNAL_SERVER_ERROR", + message: + error instanceof Error + ? error.message + : "Failed to read current Texas election data", + cause: error, + }); + } + }), + /** * Get live California statewide election results (Secretary of State feed). * Defaults to the marquee races (governor + secretary of state) when no @@ -140,4 +161,40 @@ export const civicRouter = { }); } }), + + /** Authoritative current-cycle NC candidates, referenda, and results by county. */ + getNcElectionData: publicProcedure + .input( + z.object({ + county: z.string().trim().min(2).max(100), + electionDate: z.iso.date(), + civicContests: z + .array( + z.object({ + title: z.string().trim().min(1).max(300), + candidates: z + .array(z.string().trim().min(1).max(200)) + .max(50) + .optional(), + }), + ) + .max(100) + .optional(), + includePrecincts: z.boolean().optional(), + }), + ) + .query(async ({ input }) => { + try { + return await getCurrentNcElectionData(input); + } catch (error) { + throw new TRPCError({ + code: "BAD_REQUEST", + message: + error instanceof Error + ? error.message + : "Failed to read NCSBE election data", + cause: error, + }); + } + }), } satisfies TRPCRouterRecord; diff --git a/packages/api/src/router/content.ts b/packages/api/src/router/content.ts index 817947db..102ba209 100644 --- a/packages/api/src/router/content.ts +++ b/packages/api/src/router/content.ts @@ -185,6 +185,59 @@ const _ContentDetailSchema = ContentCardSchema.extend({ export type ContentDetail = z.infer; export const contentRouter = { + // Read the latest Texas bulk session persisted by the scraper. This route is + // intentionally current-session only; it is not a historical session browser. + texasBills: publicProcedure + .input( + z + .object({ + limit: z.number().int().min(1).max(50).default(20), + cursor: z.number().int().min(0).default(0), + }) + .optional(), + ) + .query(async ({ input }) => { + const jurisdiction = "ocd-jurisdiction/country:us/state:tx/government"; + const [latest] = await db + .select({ legislativeSession: Bill.legislativeSession }) + .from(Bill) + .where(eq(Bill.jurisdiction, jurisdiction)) + .orderBy(desc(Bill.updatedAt), desc(Bill.createdAt)) + .limit(1); + if (!latest) return { items: [], nextCursor: undefined }; + const limit = input?.limit ?? 20; + const cursor = input?.cursor ?? 0; + const rows = await db + .select({ + id: Bill.id, + billNumber: Bill.billNumber, + title: Bill.title, + description: Bill.description, + summary: Bill.summary, + status: Bill.status, + chamber: Bill.chamber, + legislativeSession: Bill.legislativeSession, + openStatesId: Bill.openStatesId, + subjects: Bill.subjects, + updatedAt: Bill.updatedAt, + }) + .from(Bill) + .where( + and( + eq(Bill.jurisdiction, jurisdiction), + eq(Bill.legislativeSession, latest.legislativeSession), + ), + ) + .orderBy(desc(Bill.updatedAt), desc(Bill.billNumber)) + .limit(limit + 1) + .offset(cursor); + const hasMore = rows.length > limit; + return { + items: hasMore ? rows.slice(0, limit) : rows, + nextCursor: hasMore ? cursor + limit : undefined, + }; + }), + // Get all content from database getAll: publicProcedure.query(async () => { const bills = await db @@ -587,6 +640,13 @@ export const contentRouter = { type?: string; }[], status: b.status ?? undefined, + jurisdiction: b.jurisdiction, + legislativeSession: b.legislativeSession || undefined, + openStatesId: b.openStatesId ?? undefined, + subjects: b.subjects ?? [], + sponsorships: b.sponsorships ?? [], + documents: b.documents ?? [], + votes: b.votes ?? [], lensData: await getLensData(b.id, "bill"), }, ]); diff --git a/packages/api/src/router/local-government.ts b/packages/api/src/router/local-government.ts new file mode 100644 index 00000000..553f6977 --- /dev/null +++ b/packages/api/src/router/local-government.ts @@ -0,0 +1,64 @@ +import type { TRPCRouterRecord } from "@trpc/server"; +import { TRPCError } from "@trpc/server"; +import { z } from "zod/v4"; + +import { + getLocalGovernmentMeeting, + getLocalGovernmentMeetings, +} from "../lib/local-government"; +import { publicProcedure } from "../trpc"; + +const meetingListInput = z + .object({ + jurisdiction: z.string().trim().min(1).optional(), + start: z.date().optional(), + end: z.date().optional(), + limit: z.number().int().min(1).max(100).default(50), + daysAhead: z.number().int().min(1).max(180).optional(), + includePastDays: z.number().int().min(0).max(90).optional(), + }) + .optional(); + +function resolveMeetingRange(input: z.infer) { + if (!input?.daysAhead && input?.includePastDays === undefined) { + return { start: input?.start, end: input?.end }; + } + const start = input.start ?? new Date(); + start.setDate(start.getDate() - (input.includePastDays ?? 0)); + const end = input.end ?? new Date(); + end.setDate(end.getDate() + (input.daysAhead ?? 90)); + return { start, end }; +} + +async function listMeetings(input: z.infer) { + const range = resolveMeetingRange(input); + return getLocalGovernmentMeetings({ + jurisdiction: input?.jurisdiction, + start: range.start, + end: range.end, + limit: input?.limit ?? 50, + }); +} + +async function getMeeting(id: string) { + const meeting = await getLocalGovernmentMeeting(id); + if (!meeting) { + throw new TRPCError({ code: "NOT_FOUND", message: "Meeting not found" }); + } + return meeting; +} + +export const localGovernmentRouter = { + meetings: publicProcedure + .input(meetingListInput) + .query(({ input }) => listMeetings(input)), + listMeetings: publicProcedure + .input(meetingListInput) + .query(({ input }) => listMeetings(input)), + meeting: publicProcedure + .input(z.object({ id: z.uuid() })) + .query(({ input }) => getMeeting(input.id)), + getMeeting: publicProcedure + .input(z.object({ id: z.uuid() })) + .query(({ input }) => getMeeting(input.id)), +} satisfies TRPCRouterRecord; diff --git a/packages/db/drizzle/0002_shocking_morbius.sql b/packages/db/drizzle/0002_shocking_morbius.sql new file mode 100644 index 00000000..2c60297c --- /dev/null +++ b/packages/db/drizzle/0002_shocking_morbius.sql @@ -0,0 +1,186 @@ +CREATE TABLE "election_candidate" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL, + "source_id" uuid NOT NULL, + "election_date" date NOT NULL, + "county" varchar(100) NOT NULL, + "contest" text NOT NULL, + "name" text NOT NULL, + "party" varchar(30), + "vote_for" integer, + "term_years" integer, + "has_primary" boolean, + "is_partisan" boolean, + CONSTRAINT "election_candidate_sourceId_county_contest_name_party_unique" UNIQUE("source_id","county","contest","name","party") +); +--> statement-breakpoint +CREATE TABLE "election_referendum" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL, + "source_id" uuid NOT NULL, + "election_date" date NOT NULL, + "county" varchar(100) NOT NULL, + "contest" text NOT NULL, + "choice" text NOT NULL, + "description" text, + CONSTRAINT "election_referendum_sourceId_county_contest_choice_unique" UNIQUE("source_id","county","contest","choice") +); +--> statement-breakpoint +CREATE TABLE "election_result" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL, + "source_id" uuid NOT NULL, + "election_date" date NOT NULL, + "county" varchar(100) NOT NULL, + "precinct" varchar(100) NOT NULL, + "contest_id" varchar(100), + "contest_type" varchar(30), + "contest" text NOT NULL, + "choice" text NOT NULL, + "party" varchar(30), + "vote_for" integer, + "election_day_votes" integer NOT NULL, + "early_voting_votes" integer NOT NULL, + "absentee_mail_votes" integer NOT NULL, + "provisional_votes" integer NOT NULL, + "total_votes" integer NOT NULL, + "real_precinct" boolean, + CONSTRAINT "election_result_sourceId_county_precinct_contest_choice_unique" UNIQUE("source_id","county","precinct","contest","choice") +); +--> statement-breakpoint +CREATE TABLE "election_source" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL, + "provider" varchar(50) NOT NULL, + "source_kind" varchar(30) NOT NULL, + "election_date" date NOT NULL, + "source_url" text NOT NULL, + "checksum" varchar(64) NOT NULL, + "structure_version" varchar(50) NOT NULL, + "certification_status" varchar(30) DEFAULT 'unknown' NOT NULL, + "fetched_at" timestamp with time zone NOT NULL, + "created_at" timestamp DEFAULT now() NOT NULL, + "updated_at" timestamp with time zone, + CONSTRAINT "election_source_provider_sourceKind_electionDate_sourceUrl_unique" UNIQUE("provider","source_kind","election_date","source_url") +); +--> statement-breakpoint +CREATE TABLE "election_source_snapshot" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL, + "jurisdiction" varchar(20) NOT NULL, + "cycle_year" integer NOT NULL, + "provider" varchar(50) NOT NULL, + "scope" varchar(50) DEFAULT 'current' NOT NULL, + "source_version" varchar(150) NOT NULL, + "content_hash" varchar(64) NOT NULL, + "data" jsonb NOT NULL, + "diagnostics" jsonb DEFAULT '[]'::jsonb NOT NULL, + "source_urls" jsonb DEFAULT '[]'::jsonb NOT NULL, + "fetched_at" timestamp with time zone NOT NULL, + "created_at" timestamp DEFAULT now() NOT NULL, + "updated_at" timestamp with time zone, + CONSTRAINT "election_source_snapshot_jurisdiction_cycleYear_provider_scope_unique" UNIQUE("jurisdiction","cycle_year","provider","scope") +); +--> statement-breakpoint +CREATE TABLE "local_government_agenda_item" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL, + "meeting_id" uuid NOT NULL, + "external_id" varchar(128) NOT NULL, + "sequence" integer NOT NULL, + "item_number" varchar(50), + "section" varchar(100), + "item_type" varchar(50) NOT NULL, + "title" text NOT NULL, + "description" text, + "minutes_note" text, + "consent" boolean DEFAULT false NOT NULL, + "action" text, + "motion" text, + "outcome" varchar(100), + "vote_summary" text, + "mover" varchar(256), + "seconder" varchar(256), + "source_version" varchar(100), + "content_hash" varchar(64), + "source_updated_at" timestamp with time zone, + "source_url" text NOT NULL, + "created_at" timestamp DEFAULT now() NOT NULL, + "updated_at" timestamp with time zone, + CONSTRAINT "local_government_agenda_item_meetingId_externalId_unique" UNIQUE("meeting_id","external_id") +); +--> statement-breakpoint +CREATE TABLE "local_government_document" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL, + "meeting_id" uuid NOT NULL, + "type" varchar(30) NOT NULL, + "title" text NOT NULL, + "url" text NOT NULL, + "media_type" varchar(100), + "checksum" varchar(64), + "extracted_text" text, + "is_current" boolean DEFAULT true NOT NULL, + "discovered_at" timestamp with time zone DEFAULT now() NOT NULL, + "fetched_at" timestamp with time zone, + "created_at" timestamp DEFAULT now() NOT NULL, + "updated_at" timestamp with time zone, + CONSTRAINT "local_government_document_meetingId_type_url_unique" UNIQUE("meeting_id","type","url") +); +--> statement-breakpoint +CREATE TABLE "local_government_meeting" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL, + "source" varchar(50) NOT NULL, + "source_version" varchar(100) NOT NULL, + "jurisdiction" varchar(100) NOT NULL, + "governing_body" varchar(256) NOT NULL, + "external_id" varchar(128) NOT NULL, + "title" text NOT NULL, + "meeting_type" varchar(50) NOT NULL, + "status" varchar(50) NOT NULL, + "starts_at" timestamp with time zone NOT NULL, + "timezone" varchar(64), + "location" text, + "is_cancelled" boolean DEFAULT false NOT NULL, + "is_amended" boolean DEFAULT false NOT NULL, + "canonical_url" text NOT NULL, + "video_url" text, + "content_hash" varchar(64) NOT NULL, + "source_updated_at" timestamp with time zone, + "fetched_at" timestamp with time zone DEFAULT now() NOT NULL, + "created_at" timestamp DEFAULT now() NOT NULL, + "updated_at" timestamp with time zone, + CONSTRAINT "local_government_meeting_source_jurisdiction_externalId_unique" UNIQUE("source","jurisdiction","external_id") +); +--> statement-breakpoint +CREATE TABLE "local_government_vote" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL, + "agenda_item_id" uuid NOT NULL, + "external_id" varchar(128), + "voter_external_id" varchar(128), + "voter_name" varchar(256) NOT NULL, + "value" varchar(50) NOT NULL, + "sort" integer DEFAULT 0 NOT NULL, + "source_updated_at" timestamp with time zone, + "fetched_at" timestamp with time zone DEFAULT now() NOT NULL, + "created_at" timestamp DEFAULT now() NOT NULL, + CONSTRAINT "local_government_vote_agendaItemId_voterName_unique" UNIQUE("agenda_item_id","voter_name") +); +--> statement-breakpoint +ALTER TABLE "bill" DROP CONSTRAINT "bill_billNumber_sourceWebsite_unique";--> statement-breakpoint +ALTER TABLE "bill" ADD COLUMN "jurisdiction" varchar(100) DEFAULT 'ocd-jurisdiction/country:us/government' NOT NULL;--> statement-breakpoint +ALTER TABLE "bill" ADD COLUMN "legislative_session" varchar(20) DEFAULT '' NOT NULL;--> statement-breakpoint +ALTER TABLE "bill" ADD COLUMN "open_states_id" text;--> statement-breakpoint +ALTER TABLE "bill" ADD COLUMN "subjects" jsonb DEFAULT '[]'::jsonb;--> statement-breakpoint +ALTER TABLE "bill" ADD COLUMN "sponsorships" jsonb DEFAULT '[]'::jsonb;--> statement-breakpoint +ALTER TABLE "bill" ADD COLUMN "documents" jsonb DEFAULT '[]'::jsonb;--> statement-breakpoint +ALTER TABLE "bill" ADD COLUMN "votes" jsonb DEFAULT '[]'::jsonb;--> statement-breakpoint +ALTER TABLE "election_candidate" ADD CONSTRAINT "election_candidate_source_id_election_source_id_fk" FOREIGN KEY ("source_id") REFERENCES "public"."election_source"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "election_referendum" ADD CONSTRAINT "election_referendum_source_id_election_source_id_fk" FOREIGN KEY ("source_id") REFERENCES "public"."election_source"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "election_result" ADD CONSTRAINT "election_result_source_id_election_source_id_fk" FOREIGN KEY ("source_id") REFERENCES "public"."election_source"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "local_government_agenda_item" ADD CONSTRAINT "local_government_agenda_item_meeting_id_local_government_meeting_id_fk" FOREIGN KEY ("meeting_id") REFERENCES "public"."local_government_meeting"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "local_government_document" ADD CONSTRAINT "local_government_document_meeting_id_local_government_meeting_id_fk" FOREIGN KEY ("meeting_id") REFERENCES "public"."local_government_meeting"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "local_government_vote" ADD CONSTRAINT "local_government_vote_agenda_item_id_local_government_agenda_item_id_fk" FOREIGN KEY ("agenda_item_id") REFERENCES "public"."local_government_agenda_item"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +CREATE INDEX "election_candidate_lookup_idx" ON "election_candidate" USING btree ("election_date","county");--> statement-breakpoint +CREATE INDEX "election_referendum_lookup_idx" ON "election_referendum" USING btree ("election_date","county");--> statement-breakpoint +CREATE INDEX "election_result_lookup_idx" ON "election_result" USING btree ("election_date","county");--> statement-breakpoint +CREATE INDEX "election_source_date_idx" ON "election_source" USING btree ("election_date");--> statement-breakpoint +CREATE INDEX "election_snapshot_current_cycle_idx" ON "election_source_snapshot" USING btree ("jurisdiction","scope","cycle_year");--> statement-breakpoint +CREATE INDEX "local_government_agenda_item_meeting_sequence_idx" ON "local_government_agenda_item" USING btree ("meeting_id","sequence");--> statement-breakpoint +CREATE INDEX "local_government_document_meeting_idx" ON "local_government_document" USING btree ("meeting_id");--> statement-breakpoint +CREATE INDEX "local_government_meeting_jurisdiction_date_idx" ON "local_government_meeting" USING btree ("jurisdiction","starts_at");--> statement-breakpoint +CREATE INDEX "local_government_vote_agenda_item_idx" ON "local_government_vote" USING btree ("agenda_item_id");--> statement-breakpoint +ALTER TABLE "bill" ADD CONSTRAINT "bill_billNumber_sourceWebsite_legislativeSession_unique" UNIQUE("bill_number","source_website","legislative_session"); \ No newline at end of file diff --git a/packages/db/drizzle/meta/0002_snapshot.json b/packages/db/drizzle/meta/0002_snapshot.json new file mode 100644 index 00000000..8326c076 --- /dev/null +++ b/packages/db/drizzle/meta/0002_snapshot.json @@ -0,0 +1,3959 @@ +{ + "id": "f05c1857-f819-4490-85d1-fd8fbf048d67", + "prevId": "e441bd5c-9e5d-4b08-9a31-a4c21f14b3e5", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.bill": { + "name": "bill", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "bill_number": { + "name": "bill_number", + "type": "varchar(100)", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sponsor": { + "name": "sponsor", + "type": "varchar(256)", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "varchar(100)", + "primaryKey": false, + "notNull": false + }, + "introduced_date": { + "name": "introduced_date", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "congress": { + "name": "congress", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "chamber": { + "name": "chamber", + "type": "varchar(50)", + "primaryKey": false, + "notNull": false + }, + "jurisdiction": { + "name": "jurisdiction", + "type": "varchar(100)", + "primaryKey": false, + "notNull": true, + "default": "'ocd-jurisdiction/country:us/government'" + }, + "legislative_session": { + "name": "legislative_session", + "type": "varchar(20)", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "open_states_id": { + "name": "open_states_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "subjects": { + "name": "subjects", + "type": "jsonb", + "primaryKey": false, + "notNull": false, + "default": "'[]'::jsonb" + }, + "sponsorships": { + "name": "sponsorships", + "type": "jsonb", + "primaryKey": false, + "notNull": false, + "default": "'[]'::jsonb" + }, + "documents": { + "name": "documents", + "type": "jsonb", + "primaryKey": false, + "notNull": false, + "default": "'[]'::jsonb" + }, + "votes": { + "name": "votes", + "type": "jsonb", + "primaryKey": false, + "notNull": false, + "default": "'[]'::jsonb" + }, + "summary": { + "name": "summary", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "full_text": { + "name": "full_text", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "ai_generated_article": { + "name": "ai_generated_article", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "thumbnail_url": { + "name": "thumbnail_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "images": { + "name": "images", + "type": "jsonb", + "primaryKey": false, + "notNull": false, + "default": "'[]'::jsonb" + }, + "actions": { + "name": "actions", + "type": "jsonb", + "primaryKey": false, + "notNull": false, + "default": "'[]'::jsonb" + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_website": { + "name": "source_website", + "type": "varchar(50)", + "primaryKey": false, + "notNull": true + }, + "content_hash": { + "name": "content_hash", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "versions": { + "name": "versions", + "type": "jsonb", + "primaryKey": false, + "notNull": false, + "default": "'[]'::jsonb" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "search_vector": { + "name": "search_vector", + "type": "tsvector", + "primaryKey": false, + "notNull": false, + "generated": { + "as": "(\n setweight(to_tsvector('english', coalesce(bill_number, '')), 'A') ||\n setweight(to_tsvector('english', coalesce(title, '')), 'A') ||\n setweight(to_tsvector('english', coalesce(sponsor, '')), 'B') ||\n setweight(to_tsvector('english', coalesce(summary, '') || ' ' || coalesce(description, '')), 'B') ||\n setweight(to_tsvector('english', coalesce(full_text, '')), 'C')\n )", + "type": "stored" + } + } + }, + "indexes": { + "bill_search_vector_idx": { + "name": "bill_search_vector_idx", + "columns": [ + { + "expression": "search_vector", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "gin", + "with": {} + }, + "bill_number_trgm_idx": { + "name": "bill_number_trgm_idx", + "columns": [ + { + "expression": "bill_number", + "isExpression": false, + "asc": true, + "nulls": "last", + "opclass": "gin_trgm_ops" + } + ], + "isUnique": false, + "concurrently": false, + "method": "gin", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "bill_billNumber_sourceWebsite_legislativeSession_unique": { + "name": "bill_billNumber_sourceWebsite_legislativeSession_unique", + "nullsNotDistinct": false, + "columns": ["bill_number", "source_website", "legislative_session"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.blocked_content": { + "name": "blocked_content", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "varchar(20)", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "blocked_content_user_id_idx": { + "name": "blocked_content_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "blocked_content_userId_name_type_unique": { + "name": "blocked_content_userId_name_type_unique", + "nullsNotDistinct": false, + "columns": ["user_id", "name", "type"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.candidate": { + "name": "candidate", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "contest_id": { + "name": "contest_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "party": { + "name": "party", + "type": "varchar(100)", + "primaryKey": false, + "notNull": false + }, + "candidate_url": { + "name": "candidate_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "photo_url": { + "name": "photo_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "phone": { + "name": "phone", + "type": "varchar(50)", + "primaryKey": false, + "notNull": false + }, + "incumbent": { + "name": "incumbent", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "biography": { + "name": "biography", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "candidate_contest_id_idx": { + "name": "candidate_contest_id_idx", + "columns": [ + { + "expression": "contest_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.civic_api_cache": { + "name": "civic_api_cache", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "address_hash": { + "name": "address_hash", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true + }, + "endpoint": { + "name": "endpoint", + "type": "varchar(50)", + "primaryKey": false, + "notNull": true + }, + "params": { + "name": "params", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "response_data": { + "name": "response_data", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "fetched_at": { + "name": "fetched_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "civic_cache_expires_idx": { + "name": "civic_cache_expires_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "civic_api_cache_addressHash_endpoint_params_unique": { + "name": "civic_api_cache_addressHash_endpoint_params_unique", + "nullsNotDistinct": false, + "columns": ["address_hash", "endpoint", "params"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.content_lens": { + "name": "content_lens", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "content_type": { + "name": "content_type", + "type": "varchar(20)", + "primaryKey": false, + "notNull": true + }, + "content_id": { + "name": "content_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "content_hash": { + "name": "content_hash", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true + }, + "lens_data": { + "name": "lens_data", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "model_version": { + "name": "model_version", + "type": "varchar(50)", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "content_lens_content_id_idx": { + "name": "content_lens_content_id_idx", + "columns": [ + { + "expression": "content_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "content_lens_contentType_contentId_unique": { + "name": "content_lens_contentType_contentId_unique", + "nullsNotDistinct": false, + "columns": ["content_type", "content_id"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.contest": { + "name": "contest", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "election_id": { + "name": "election_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "office": { + "name": "office", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "district_name": { + "name": "district_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "district_scope": { + "name": "district_scope", + "type": "varchar(50)", + "primaryKey": false, + "notNull": false + }, + "number_elected": { + "name": "number_elected", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 1 + }, + "referendum_title": { + "name": "referendum_title", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "referendum_subtitle": { + "name": "referendum_subtitle", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "referendum_text": { + "name": "referendum_text", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "referendum_pro_statement": { + "name": "referendum_pro_statement", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "referendum_con_statement": { + "name": "referendum_con_statement", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "referendum_url": { + "name": "referendum_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "type": { + "name": "type", + "type": "varchar(20)", + "primaryKey": false, + "notNull": true + }, + "role_description": { + "name": "role_description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "summary": { + "name": "summary", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "summary_is_ai_generated": { + "name": "summary_is_ai_generated", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "fiscal_impact": { + "name": "fiscal_impact", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "citations": { + "name": "citations", + "type": "jsonb", + "primaryKey": false, + "notNull": false, + "default": "'[]'::jsonb" + }, + "source": { + "name": "source", + "type": "varchar(50)", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "contest_election_id_idx": { + "name": "contest_election_id_idx", + "columns": [ + { + "expression": "election_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.court_case": { + "name": "court_case", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "case_number": { + "name": "case_number", + "type": "varchar(100)", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "court": { + "name": "court", + "type": "varchar(256)", + "primaryKey": false, + "notNull": true + }, + "filed_date": { + "name": "filed_date", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "varchar(100)", + "primaryKey": false, + "notNull": false + }, + "full_text": { + "name": "full_text", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "ai_generated_article": { + "name": "ai_generated_article", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "thumbnail_url": { + "name": "thumbnail_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "images": { + "name": "images", + "type": "jsonb", + "primaryKey": false, + "notNull": false, + "default": "'[]'::jsonb" + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "content_hash": { + "name": "content_hash", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "versions": { + "name": "versions", + "type": "jsonb", + "primaryKey": false, + "notNull": false, + "default": "'[]'::jsonb" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "search_vector": { + "name": "search_vector", + "type": "tsvector", + "primaryKey": false, + "notNull": false, + "generated": { + "as": "(\n setweight(to_tsvector('english', coalesce(case_number, '')), 'A') ||\n setweight(to_tsvector('english', coalesce(title, '')), 'A') ||\n setweight(to_tsvector('english', coalesce(description, '')), 'B') ||\n setweight(to_tsvector('english', coalesce(full_text, '')), 'C')\n )", + "type": "stored" + } + } + }, + "indexes": { + "court_case_search_vector_idx": { + "name": "court_case_search_vector_idx", + "columns": [ + { + "expression": "search_vector", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "gin", + "with": {} + }, + "court_case_number_trgm_idx": { + "name": "court_case_number_trgm_idx", + "columns": [ + { + "expression": "case_number", + "isExpression": false, + "asc": true, + "nulls": "last", + "opclass": "gin_trgm_ops" + } + ], + "isUnique": false, + "concurrently": false, + "method": "gin", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "court_case_caseNumber_unique": { + "name": "court_case_caseNumber_unique", + "nullsNotDistinct": false, + "columns": ["case_number"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.election_candidate": { + "name": "election_candidate", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "source_id": { + "name": "source_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "election_date": { + "name": "election_date", + "type": "date", + "primaryKey": false, + "notNull": true + }, + "county": { + "name": "county", + "type": "varchar(100)", + "primaryKey": false, + "notNull": true + }, + "contest": { + "name": "contest", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "party": { + "name": "party", + "type": "varchar(30)", + "primaryKey": false, + "notNull": false + }, + "vote_for": { + "name": "vote_for", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "term_years": { + "name": "term_years", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "has_primary": { + "name": "has_primary", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "is_partisan": { + "name": "is_partisan", + "type": "boolean", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "election_candidate_lookup_idx": { + "name": "election_candidate_lookup_idx", + "columns": [ + { + "expression": "election_date", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "county", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "election_candidate_source_id_election_source_id_fk": { + "name": "election_candidate_source_id_election_source_id_fk", + "tableFrom": "election_candidate", + "tableTo": "election_source", + "columnsFrom": ["source_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "election_candidate_sourceId_county_contest_name_party_unique": { + "name": "election_candidate_sourceId_county_contest_name_party_unique", + "nullsNotDistinct": false, + "columns": ["source_id", "county", "contest", "name", "party"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.election": { + "name": "election", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "external_id": { + "name": "external_id", + "type": "varchar(100)", + "primaryKey": false, + "notNull": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "date": { + "name": "date", + "type": "varchar(20)", + "primaryKey": false, + "notNull": true + }, + "election_type": { + "name": "election_type", + "type": "varchar(20)", + "primaryKey": false, + "notNull": true + }, + "ocd_division_id": { + "name": "ocd_division_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "source": { + "name": "source", + "type": "varchar(50)", + "primaryKey": false, + "notNull": true + }, + "deadlines": { + "name": "deadlines", + "type": "jsonb", + "primaryKey": false, + "notNull": false, + "default": "'[]'::jsonb" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "election_externalId_source_unique": { + "name": "election_externalId_source_unique", + "nullsNotDistinct": false, + "columns": ["external_id", "source"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.election_referendum": { + "name": "election_referendum", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "source_id": { + "name": "source_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "election_date": { + "name": "election_date", + "type": "date", + "primaryKey": false, + "notNull": true + }, + "county": { + "name": "county", + "type": "varchar(100)", + "primaryKey": false, + "notNull": true + }, + "contest": { + "name": "contest", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "choice": { + "name": "choice", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "election_referendum_lookup_idx": { + "name": "election_referendum_lookup_idx", + "columns": [ + { + "expression": "election_date", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "county", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "election_referendum_source_id_election_source_id_fk": { + "name": "election_referendum_source_id_election_source_id_fk", + "tableFrom": "election_referendum", + "tableTo": "election_source", + "columnsFrom": ["source_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "election_referendum_sourceId_county_contest_choice_unique": { + "name": "election_referendum_sourceId_county_contest_choice_unique", + "nullsNotDistinct": false, + "columns": ["source_id", "county", "contest", "choice"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.election_result": { + "name": "election_result", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "source_id": { + "name": "source_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "election_date": { + "name": "election_date", + "type": "date", + "primaryKey": false, + "notNull": true + }, + "county": { + "name": "county", + "type": "varchar(100)", + "primaryKey": false, + "notNull": true + }, + "precinct": { + "name": "precinct", + "type": "varchar(100)", + "primaryKey": false, + "notNull": true + }, + "contest_id": { + "name": "contest_id", + "type": "varchar(100)", + "primaryKey": false, + "notNull": false + }, + "contest_type": { + "name": "contest_type", + "type": "varchar(30)", + "primaryKey": false, + "notNull": false + }, + "contest": { + "name": "contest", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "choice": { + "name": "choice", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "party": { + "name": "party", + "type": "varchar(30)", + "primaryKey": false, + "notNull": false + }, + "vote_for": { + "name": "vote_for", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "election_day_votes": { + "name": "election_day_votes", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "early_voting_votes": { + "name": "early_voting_votes", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "absentee_mail_votes": { + "name": "absentee_mail_votes", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "provisional_votes": { + "name": "provisional_votes", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "total_votes": { + "name": "total_votes", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "real_precinct": { + "name": "real_precinct", + "type": "boolean", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "election_result_lookup_idx": { + "name": "election_result_lookup_idx", + "columns": [ + { + "expression": "election_date", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "county", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "election_result_source_id_election_source_id_fk": { + "name": "election_result_source_id_election_source_id_fk", + "tableFrom": "election_result", + "tableTo": "election_source", + "columnsFrom": ["source_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "election_result_sourceId_county_precinct_contest_choice_unique": { + "name": "election_result_sourceId_county_precinct_contest_choice_unique", + "nullsNotDistinct": false, + "columns": ["source_id", "county", "precinct", "contest", "choice"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.election_source": { + "name": "election_source", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "provider": { + "name": "provider", + "type": "varchar(50)", + "primaryKey": false, + "notNull": true + }, + "source_kind": { + "name": "source_kind", + "type": "varchar(30)", + "primaryKey": false, + "notNull": true + }, + "election_date": { + "name": "election_date", + "type": "date", + "primaryKey": false, + "notNull": true + }, + "source_url": { + "name": "source_url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "checksum": { + "name": "checksum", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true + }, + "structure_version": { + "name": "structure_version", + "type": "varchar(50)", + "primaryKey": false, + "notNull": true + }, + "certification_status": { + "name": "certification_status", + "type": "varchar(30)", + "primaryKey": false, + "notNull": true, + "default": "'unknown'" + }, + "fetched_at": { + "name": "fetched_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "election_source_date_idx": { + "name": "election_source_date_idx", + "columns": [ + { + "expression": "election_date", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "election_source_provider_sourceKind_electionDate_sourceUrl_unique": { + "name": "election_source_provider_sourceKind_electionDate_sourceUrl_unique", + "nullsNotDistinct": false, + "columns": ["provider", "source_kind", "election_date", "source_url"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.election_source_snapshot": { + "name": "election_source_snapshot", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "jurisdiction": { + "name": "jurisdiction", + "type": "varchar(20)", + "primaryKey": false, + "notNull": true + }, + "cycle_year": { + "name": "cycle_year", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "varchar(50)", + "primaryKey": false, + "notNull": true + }, + "scope": { + "name": "scope", + "type": "varchar(50)", + "primaryKey": false, + "notNull": true, + "default": "'current'" + }, + "source_version": { + "name": "source_version", + "type": "varchar(150)", + "primaryKey": false, + "notNull": true + }, + "content_hash": { + "name": "content_hash", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true + }, + "data": { + "name": "data", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "diagnostics": { + "name": "diagnostics", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "source_urls": { + "name": "source_urls", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "fetched_at": { + "name": "fetched_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "election_snapshot_current_cycle_idx": { + "name": "election_snapshot_current_cycle_idx", + "columns": [ + { + "expression": "jurisdiction", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "scope", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "cycle_year", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "election_source_snapshot_jurisdiction_cycleYear_provider_scope_unique": { + "name": "election_source_snapshot_jurisdiction_cycleYear_provider_scope_unique", + "nullsNotDistinct": false, + "columns": ["jurisdiction", "cycle_year", "provider", "scope"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.government_content": { + "name": "government_content", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "varchar(50)", + "primaryKey": false, + "notNull": true + }, + "published_date": { + "name": "published_date", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "full_text": { + "name": "full_text", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "ai_generated_article": { + "name": "ai_generated_article", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "thumbnail_url": { + "name": "thumbnail_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "images": { + "name": "images", + "type": "jsonb", + "primaryKey": false, + "notNull": false, + "default": "'[]'::jsonb" + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source": { + "name": "source", + "type": "varchar(100)", + "primaryKey": false, + "notNull": true, + "default": "'whitehouse.gov'" + }, + "content_hash": { + "name": "content_hash", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "versions": { + "name": "versions", + "type": "jsonb", + "primaryKey": false, + "notNull": false, + "default": "'[]'::jsonb" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "search_vector": { + "name": "search_vector", + "type": "tsvector", + "primaryKey": false, + "notNull": false, + "generated": { + "as": "(\n setweight(to_tsvector('english', coalesce(title, '')), 'A') ||\n setweight(to_tsvector('english', coalesce(description, '')), 'B') ||\n setweight(to_tsvector('english', coalesce(full_text, '')), 'C')\n )", + "type": "stored" + } + } + }, + "indexes": { + "government_content_search_vector_idx": { + "name": "government_content_search_vector_idx", + "columns": [ + { + "expression": "search_vector", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "gin", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "government_content_url_unique": { + "name": "government_content_url_unique", + "nullsNotDistinct": false, + "columns": ["url"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.legistar_agenda_item": { + "name": "legistar_agenda_item", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "jurisdiction": { + "name": "jurisdiction", + "type": "varchar(50)", + "primaryKey": false, + "notNull": true + }, + "event_item_id": { + "name": "event_item_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "event_id": { + "name": "event_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "agenda_sequence": { + "name": "agenda_sequence", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "agenda_number": { + "name": "agenda_number", + "type": "varchar(50)", + "primaryKey": false, + "notNull": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "action_name": { + "name": "action_name", + "type": "varchar(256)", + "primaryKey": false, + "notNull": false + }, + "passed_flag_name": { + "name": "passed_flag_name", + "type": "varchar(50)", + "primaryKey": false, + "notNull": false + }, + "tally": { + "name": "tally", + "type": "varchar(50)", + "primaryKey": false, + "notNull": false + }, + "mover_name": { + "name": "mover_name", + "type": "varchar(256)", + "primaryKey": false, + "notNull": false + }, + "seconder_name": { + "name": "seconder_name", + "type": "varchar(256)", + "primaryKey": false, + "notNull": false + }, + "matter_id": { + "name": "matter_id", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "matter_file": { + "name": "matter_file", + "type": "varchar(100)", + "primaryKey": false, + "notNull": false + }, + "matter_name": { + "name": "matter_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "matter_type": { + "name": "matter_type", + "type": "varchar(100)", + "primaryKey": false, + "notNull": false + }, + "matter_status": { + "name": "matter_status", + "type": "varchar(100)", + "primaryKey": false, + "notNull": false + }, + "consent": { + "name": "consent", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "agenda_note": { + "name": "agenda_note", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "minutes_note": { + "name": "minutes_note", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_modified_utc": { + "name": "last_modified_utc", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "fetched_at": { + "name": "fetched_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "legistar_agenda_item_event_idx": { + "name": "legistar_agenda_item_event_idx", + "columns": [ + { + "expression": "event_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "legistar_agenda_item_jurisdiction_eventItemId_unique": { + "name": "legistar_agenda_item_jurisdiction_eventItemId_unique", + "nullsNotDistinct": false, + "columns": ["jurisdiction", "event_item_id"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.legistar_body": { + "name": "legistar_body", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "jurisdiction": { + "name": "jurisdiction", + "type": "varchar(50)", + "primaryKey": false, + "notNull": true + }, + "body_id": { + "name": "body_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "body_guid": { + "name": "body_guid", + "type": "varchar(100)", + "primaryKey": false, + "notNull": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "type_name": { + "name": "type_name", + "type": "varchar(100)", + "primaryKey": false, + "notNull": false + }, + "active_flag": { + "name": "active_flag", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": true + }, + "number_of_members": { + "name": "number_of_members", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "contact_name": { + "name": "contact_name", + "type": "varchar(256)", + "primaryKey": false, + "notNull": false + }, + "contact_email": { + "name": "contact_email", + "type": "varchar(256)", + "primaryKey": false, + "notNull": false + }, + "contact_phone": { + "name": "contact_phone", + "type": "varchar(50)", + "primaryKey": false, + "notNull": false + }, + "fetched_at": { + "name": "fetched_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "legistar_body_jurisdiction_bodyId_unique": { + "name": "legistar_body_jurisdiction_bodyId_unique", + "nullsNotDistinct": false, + "columns": ["jurisdiction", "body_id"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.legistar_matter": { + "name": "legistar_matter", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "jurisdiction": { + "name": "jurisdiction", + "type": "varchar(50)", + "primaryKey": false, + "notNull": true + }, + "matter_id": { + "name": "matter_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "matter_guid": { + "name": "matter_guid", + "type": "varchar(100)", + "primaryKey": false, + "notNull": false + }, + "matter_file": { + "name": "matter_file", + "type": "varchar(100)", + "primaryKey": false, + "notNull": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "type_name": { + "name": "type_name", + "type": "varchar(100)", + "primaryKey": false, + "notNull": false + }, + "status_name": { + "name": "status_name", + "type": "varchar(100)", + "primaryKey": false, + "notNull": false + }, + "body_name": { + "name": "body_name", + "type": "varchar(256)", + "primaryKey": false, + "notNull": false + }, + "body_id": { + "name": "body_id", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "intro_date": { + "name": "intro_date", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "agenda_date": { + "name": "agenda_date", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "passed_date": { + "name": "passed_date", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "enactment_date": { + "name": "enactment_date", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "enactment_number": { + "name": "enactment_number", + "type": "varchar(100)", + "primaryKey": false, + "notNull": false + }, + "requester": { + "name": "requester", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "notes": { + "name": "notes", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_modified_utc": { + "name": "last_modified_utc", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "fetched_at": { + "name": "fetched_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "legistar_matter_file_idx": { + "name": "legistar_matter_file_idx", + "columns": [ + { + "expression": "matter_file", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "legistar_matter_jurisdiction_matterId_unique": { + "name": "legistar_matter_jurisdiction_matterId_unique", + "nullsNotDistinct": false, + "columns": ["jurisdiction", "matter_id"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.legistar_meeting": { + "name": "legistar_meeting", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "jurisdiction": { + "name": "jurisdiction", + "type": "varchar(50)", + "primaryKey": false, + "notNull": true + }, + "event_id": { + "name": "event_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "event_guid": { + "name": "event_guid", + "type": "varchar(100)", + "primaryKey": false, + "notNull": false + }, + "body_id": { + "name": "body_id", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "body_name": { + "name": "body_name", + "type": "varchar(256)", + "primaryKey": false, + "notNull": false + }, + "date": { + "name": "date", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "time": { + "name": "time", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "location": { + "name": "location", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "agenda_file": { + "name": "agenda_file", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "minutes_file": { + "name": "minutes_file", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "video_path": { + "name": "video_path", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "agenda_status_name": { + "name": "agenda_status_name", + "type": "varchar(100)", + "primaryKey": false, + "notNull": false + }, + "minutes_status_name": { + "name": "minutes_status_name", + "type": "varchar(100)", + "primaryKey": false, + "notNull": false + }, + "comment": { + "name": "comment", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "in_site_url": { + "name": "in_site_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_modified_utc": { + "name": "last_modified_utc", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "fetched_at": { + "name": "fetched_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "legistar_meeting_date_idx": { + "name": "legistar_meeting_date_idx", + "columns": [ + { + "expression": "date", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "legistar_meeting_jurisdiction_eventId_unique": { + "name": "legistar_meeting_jurisdiction_eventId_unique", + "nullsNotDistinct": false, + "columns": ["jurisdiction", "event_id"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.legistar_vote": { + "name": "legistar_vote", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "jurisdiction": { + "name": "jurisdiction", + "type": "varchar(50)", + "primaryKey": false, + "notNull": true + }, + "vote_id": { + "name": "vote_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "event_item_id": { + "name": "event_item_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "person_id": { + "name": "person_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "person_name": { + "name": "person_name", + "type": "varchar(256)", + "primaryKey": false, + "notNull": true + }, + "value_name": { + "name": "value_name", + "type": "varchar(50)", + "primaryKey": false, + "notNull": true + }, + "sort": { + "name": "sort", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "last_modified_utc": { + "name": "last_modified_utc", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "fetched_at": { + "name": "fetched_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "legistar_vote_event_item_idx": { + "name": "legistar_vote_event_item_idx", + "columns": [ + { + "expression": "event_item_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "legistar_vote_person_idx": { + "name": "legistar_vote_person_idx", + "columns": [ + { + "expression": "person_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "legistar_vote_jurisdiction_voteId_unique": { + "name": "legistar_vote_jurisdiction_voteId_unique", + "nullsNotDistinct": false, + "columns": ["jurisdiction", "vote_id"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.local_government_agenda_item": { + "name": "local_government_agenda_item", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "meeting_id": { + "name": "meeting_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "external_id": { + "name": "external_id", + "type": "varchar(128)", + "primaryKey": false, + "notNull": true + }, + "sequence": { + "name": "sequence", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "item_number": { + "name": "item_number", + "type": "varchar(50)", + "primaryKey": false, + "notNull": false + }, + "section": { + "name": "section", + "type": "varchar(100)", + "primaryKey": false, + "notNull": false + }, + "item_type": { + "name": "item_type", + "type": "varchar(50)", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "minutes_note": { + "name": "minutes_note", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "consent": { + "name": "consent", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "action": { + "name": "action", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "motion": { + "name": "motion", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "outcome": { + "name": "outcome", + "type": "varchar(100)", + "primaryKey": false, + "notNull": false + }, + "vote_summary": { + "name": "vote_summary", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "mover": { + "name": "mover", + "type": "varchar(256)", + "primaryKey": false, + "notNull": false + }, + "seconder": { + "name": "seconder", + "type": "varchar(256)", + "primaryKey": false, + "notNull": false + }, + "source_version": { + "name": "source_version", + "type": "varchar(100)", + "primaryKey": false, + "notNull": false + }, + "content_hash": { + "name": "content_hash", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false + }, + "source_updated_at": { + "name": "source_updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "source_url": { + "name": "source_url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "local_government_agenda_item_meeting_sequence_idx": { + "name": "local_government_agenda_item_meeting_sequence_idx", + "columns": [ + { + "expression": "meeting_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "sequence", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "local_government_agenda_item_meeting_id_local_government_meeting_id_fk": { + "name": "local_government_agenda_item_meeting_id_local_government_meeting_id_fk", + "tableFrom": "local_government_agenda_item", + "tableTo": "local_government_meeting", + "columnsFrom": ["meeting_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "local_government_agenda_item_meetingId_externalId_unique": { + "name": "local_government_agenda_item_meetingId_externalId_unique", + "nullsNotDistinct": false, + "columns": ["meeting_id", "external_id"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.local_government_document": { + "name": "local_government_document", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "meeting_id": { + "name": "meeting_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "varchar(30)", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "media_type": { + "name": "media_type", + "type": "varchar(100)", + "primaryKey": false, + "notNull": false + }, + "checksum": { + "name": "checksum", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false + }, + "extracted_text": { + "name": "extracted_text", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "is_current": { + "name": "is_current", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "discovered_at": { + "name": "discovered_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "fetched_at": { + "name": "fetched_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "local_government_document_meeting_idx": { + "name": "local_government_document_meeting_idx", + "columns": [ + { + "expression": "meeting_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "local_government_document_meeting_id_local_government_meeting_id_fk": { + "name": "local_government_document_meeting_id_local_government_meeting_id_fk", + "tableFrom": "local_government_document", + "tableTo": "local_government_meeting", + "columnsFrom": ["meeting_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "local_government_document_meetingId_type_url_unique": { + "name": "local_government_document_meetingId_type_url_unique", + "nullsNotDistinct": false, + "columns": ["meeting_id", "type", "url"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.local_government_meeting": { + "name": "local_government_meeting", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "source": { + "name": "source", + "type": "varchar(50)", + "primaryKey": false, + "notNull": true + }, + "source_version": { + "name": "source_version", + "type": "varchar(100)", + "primaryKey": false, + "notNull": true + }, + "jurisdiction": { + "name": "jurisdiction", + "type": "varchar(100)", + "primaryKey": false, + "notNull": true + }, + "governing_body": { + "name": "governing_body", + "type": "varchar(256)", + "primaryKey": false, + "notNull": true + }, + "external_id": { + "name": "external_id", + "type": "varchar(128)", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "meeting_type": { + "name": "meeting_type", + "type": "varchar(50)", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "varchar(50)", + "primaryKey": false, + "notNull": true + }, + "starts_at": { + "name": "starts_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "timezone": { + "name": "timezone", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false + }, + "location": { + "name": "location", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "is_cancelled": { + "name": "is_cancelled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "is_amended": { + "name": "is_amended", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "canonical_url": { + "name": "canonical_url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "video_url": { + "name": "video_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "content_hash": { + "name": "content_hash", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true + }, + "source_updated_at": { + "name": "source_updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "fetched_at": { + "name": "fetched_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "local_government_meeting_jurisdiction_date_idx": { + "name": "local_government_meeting_jurisdiction_date_idx", + "columns": [ + { + "expression": "jurisdiction", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "starts_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "local_government_meeting_source_jurisdiction_externalId_unique": { + "name": "local_government_meeting_source_jurisdiction_externalId_unique", + "nullsNotDistinct": false, + "columns": ["source", "jurisdiction", "external_id"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.local_government_vote": { + "name": "local_government_vote", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "agenda_item_id": { + "name": "agenda_item_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "external_id": { + "name": "external_id", + "type": "varchar(128)", + "primaryKey": false, + "notNull": false + }, + "voter_external_id": { + "name": "voter_external_id", + "type": "varchar(128)", + "primaryKey": false, + "notNull": false + }, + "voter_name": { + "name": "voter_name", + "type": "varchar(256)", + "primaryKey": false, + "notNull": true + }, + "value": { + "name": "value", + "type": "varchar(50)", + "primaryKey": false, + "notNull": true + }, + "sort": { + "name": "sort", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "source_updated_at": { + "name": "source_updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "fetched_at": { + "name": "fetched_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "local_government_vote_agenda_item_idx": { + "name": "local_government_vote_agenda_item_idx", + "columns": [ + { + "expression": "agenda_item_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "local_government_vote_agenda_item_id_local_government_agenda_item_id_fk": { + "name": "local_government_vote_agenda_item_id_local_government_agenda_item_id_fk", + "tableFrom": "local_government_vote", + "tableTo": "local_government_agenda_item", + "columnsFrom": ["agenda_item_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "local_government_vote_agendaItemId_voterName_unique": { + "name": "local_government_vote_agendaItemId_voterName_unique", + "nullsNotDistinct": false, + "columns": ["agenda_item_id", "voter_name"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.polling_location": { + "name": "polling_location", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "election_id": { + "name": "election_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "address_line1": { + "name": "address_line1", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "address_line2": { + "name": "address_line2", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "city": { + "name": "city", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "state": { + "name": "state", + "type": "varchar(2)", + "primaryKey": false, + "notNull": true + }, + "zip": { + "name": "zip", + "type": "varchar(10)", + "primaryKey": false, + "notNull": true + }, + "hours": { + "name": "hours", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "latitude": { + "name": "latitude", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "longitude": { + "name": "longitude", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "location_type": { + "name": "location_type", + "type": "varchar(20)", + "primaryKey": false, + "notNull": true + }, + "voter_services": { + "name": "voter_services", + "type": "jsonb", + "primaryKey": false, + "notNull": false, + "default": "'[]'::jsonb" + }, + "start_date": { + "name": "start_date", + "type": "varchar(20)", + "primaryKey": false, + "notNull": false + }, + "end_date": { + "name": "end_date", + "type": "varchar(20)", + "primaryKey": false, + "notNull": false + }, + "source": { + "name": "source", + "type": "varchar(50)", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "polling_location_election_id_idx": { + "name": "polling_location_election_id_idx", + "columns": [ + { + "expression": "election_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.post": { + "name": "post", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "title": { + "name": "title", + "type": "varchar(256)", + "primaryKey": false, + "notNull": true + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.role_description": { + "name": "role_description", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "role": { + "name": "role", + "type": "varchar(50)", + "primaryKey": false, + "notNull": true + }, + "level": { + "name": "level", + "type": "varchar(50)", + "primaryKey": false, + "notNull": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source": { + "name": "source", + "type": "varchar(20)", + "primaryKey": false, + "notNull": true, + "default": "'seed'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "role_description_role_level_unique": { + "name": "role_description_role_level_unique", + "nullsNotDistinct": false, + "columns": ["role", "level"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.saved_article": { + "name": "saved_article", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "content_id": { + "name": "content_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "content_type": { + "name": "content_type", + "type": "varchar(20)", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "saved_article_user_id_idx": { + "name": "saved_article_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "saved_article_userId_contentId_unique": { + "name": "saved_article_userId_contentId_unique", + "nullsNotDistinct": false, + "columns": ["user_id", "content_id"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user_preference": { + "name": "user_preference", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "topics": { + "name": "topics", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "content_types": { + "name": "content_types", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "user_preference_userId_unique": { + "name": "user_preference_userId_unique", + "nullsNotDistinct": false, + "columns": ["user_id"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user_settings": { + "name": "user_settings", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "location": { + "name": "location", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "personalize": { + "name": "personalize", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "analytics": { + "name": "analytics", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "crash": { + "name": "crash", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "offline": { + "name": "offline", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "user_settings_userId_unique": { + "name": "user_settings_userId_unique", + "nullsNotDistinct": false, + "columns": ["user_id"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.video": { + "name": "video", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "content_type": { + "name": "content_type", + "type": "varchar(20)", + "primaryKey": false, + "notNull": true + }, + "content_id": { + "name": "content_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "varchar(100)", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "image_data": { + "name": "image_data", + "type": "bytea", + "primaryKey": false, + "notNull": false + }, + "image_mime_type": { + "name": "image_mime_type", + "type": "varchar(50)", + "primaryKey": false, + "notNull": false + }, + "image_width": { + "name": "image_width", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "image_height": { + "name": "image_height", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "thumbnail_url": { + "name": "thumbnail_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "author": { + "name": "author", + "type": "varchar(100)", + "primaryKey": false, + "notNull": false + }, + "engagement_metrics": { + "name": "engagement_metrics", + "type": "jsonb", + "primaryKey": false, + "notNull": false, + "default": "'{\"likes\":0,\"comments\":0,\"shares\":0}'::jsonb" + }, + "source_content_hash": { + "name": "source_content_hash", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "video_content_id_idx": { + "name": "video_content_id_idx", + "columns": [ + { + "expression": "content_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "video_created_at_idx": { + "name": "video_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "video_contentType_contentId_unique": { + "name": "video_contentType_contentId_unique", + "nullsNotDistinct": false, + "columns": ["content_type", "content_id"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.account": { + "name": "account", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "id_token": { + "name": "id_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "access_token_expires_at": { + "name": "access_token_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "refresh_token_expires_at": { + "name": "refresh_token_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "account_user_id_user_id_fk": { + "name": "account_user_id_user_id_fk", + "tableFrom": "account", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.session": { + "name": "session", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "session_user_id_user_id_fk": { + "name": "session_user_id_user_id_fk", + "tableFrom": "session", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "session_token_unique": { + "name": "session_token_unique", + "nullsNotDistinct": false, + "columns": ["token"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user": { + "name": "user", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email_verified": { + "name": "email_verified", + "type": "boolean", + "primaryKey": false, + "notNull": true + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "user_email_unique": { + "name": "user_email_unique", + "nullsNotDistinct": false, + "columns": ["email"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.verification": { + "name": "verification", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": {}, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} diff --git a/packages/db/drizzle/meta/_journal.json b/packages/db/drizzle/meta/_journal.json index 9a48ab91..1ffa4169 100644 --- a/packages/db/drizzle/meta/_journal.json +++ b/packages/db/drizzle/meta/_journal.json @@ -15,6 +15,13 @@ "when": 1784735846676, "tag": "0001_premium_famine", "breakpoints": true + }, + { + "idx": 2, + "version": "7", + "when": 1784736008823, + "tag": "0002_shocking_morbius", + "breakpoints": true } ] } diff --git a/packages/db/src/schema.ts b/packages/db/src/schema.ts index 3c990bae..0f009387 100644 --- a/packages/db/src/schema.ts +++ b/packages/db/src/schema.ts @@ -56,6 +56,52 @@ export const Bill = pgTable( introducedDate: t.timestamp(), congress: t.integer(), // e.g., 118 for 118th Congress chamber: t.varchar({ length: 50 }), // "House" or "Senate" + jurisdiction: t + .varchar({ length: 100 }) + .notNull() + .default("ocd-jurisdiction/country:us/government"), + legislativeSession: t.varchar({ length: 20 }).notNull().default(""), + openStatesId: t.text(), + subjects: t.jsonb().$type().default([]), + sponsorships: t + .jsonb() + .$type< + { + name: string; + classification: "primary" | "cosponsor"; + chamber: "House" | "Senate"; + }[] + >() + .default([]), + documents: t + .jsonb() + .$type< + { + type: "bill_text" | "analysis" | "fiscal_note"; + description: string; + htmlUrl?: string; + pdfUrl?: string; + ftpHtmlUrl?: string; + ftpPdfUrl?: string; + text?: string; + }[] + >() + .default([]), + votes: t + .jsonb() + .$type< + { + identifier: string; + date?: string; + chamber?: "House" | "Senate"; + motion?: string; + result?: string; + sourceUrl?: string; + counts: { option: string; value: number }[]; + votes: { option: string; voterName: string; openStatesId?: string }[]; + }[] + >() + .default([]), summary: t.text(), fullText: t.text(), aiGeneratedArticle: t.text(), // AI-generated accessible article version @@ -95,7 +141,11 @@ export const Bill = pgTable( ), }), (table) => ({ - uniqueBillNumberSource: unique().on(table.billNumber, table.sourceWebsite), + uniqueBillNumberSourceSession: unique().on( + table.billNumber, + table.sourceWebsite, + table.legislativeSession, + ), searchVectorIdx: index("bill_search_vector_idx").using( "gin", table.searchVector, @@ -308,6 +358,44 @@ export const ElectionRecord = pgTable( }), ); +// Provider-neutral snapshots for official election feeds that do not map +// cleanly onto the legacy relational election tables. Each provider owns one +// idempotent row per jurisdiction/cycle/scope; readers normalize and join rows +// without mixing their source attribution. +export const ElectionSourceSnapshot = pgTable( + "election_source_snapshot", + (t) => ({ + id: t.uuid().notNull().primaryKey().defaultRandom(), + jurisdiction: t.varchar({ length: 20 }).notNull(), + cycleYear: t.integer().notNull(), + provider: t.varchar({ length: 50 }).notNull(), + scope: t.varchar({ length: 50 }).notNull().default("current"), + sourceVersion: t.varchar({ length: 150 }).notNull(), + contentHash: t.varchar({ length: 64 }).notNull(), + data: t.jsonb().$type>().notNull(), + diagnostics: t.jsonb().$type().notNull().default([]), + sourceUrls: t.jsonb().$type().notNull().default([]), + fetchedAt: t.timestamp({ mode: "date", withTimezone: true }).notNull(), + createdAt: t.timestamp().defaultNow().notNull(), + updatedAt: t + .timestamp({ mode: "date", withTimezone: true }) + .$onUpdateFn(() => sql`now()`), + }), + (table) => ({ + uniqueProviderSnapshot: unique().on( + table.jurisdiction, + table.cycleYear, + table.provider, + table.scope, + ), + currentCycleIdx: index("election_snapshot_current_cycle_idx").on( + table.jurisdiction, + table.scope, + table.cycleYear, + ), + }), +); + // Role descriptions — reusable across elections, keyed by (role, level) export const RoleDescriptionRecord = pgTable( "role_description", @@ -645,6 +733,284 @@ export const LegistarVote = pgTable( }), ); +// Provider-neutral public election data. Source metadata is separated from +// normalized records so every API response can cite the exact upstream file. +// These tables intentionally exclude voter history and candidate contact/address +// fields; election scrapers should persist only public ballot/result facts. +export const ElectionSource = pgTable( + "election_source", + (t) => ({ + id: t.uuid().notNull().primaryKey().defaultRandom(), + provider: t.varchar({ length: 50 }).notNull(), + sourceKind: t.varchar({ length: 30 }).notNull(), + electionDate: t.date({ mode: "string" }).notNull(), + sourceUrl: t.text().notNull(), + checksum: t.varchar({ length: 64 }).notNull(), + structureVersion: t.varchar({ length: 50 }).notNull(), + certificationStatus: t.varchar({ length: 30 }).notNull().default("unknown"), + fetchedAt: t.timestamp({ mode: "date", withTimezone: true }).notNull(), + createdAt: t.timestamp().defaultNow().notNull(), + updatedAt: t + .timestamp({ mode: "date", withTimezone: true }) + .$onUpdateFn(() => sql`now()`), + }), + (table) => ({ + uniqueProviderFile: unique().on( + table.provider, + table.sourceKind, + table.electionDate, + table.sourceUrl, + ), + electionDateIdx: index("election_source_date_idx").on(table.electionDate), + }), +); + +export const ElectionCandidate = pgTable( + "election_candidate", + (t) => ({ + id: t.uuid().notNull().primaryKey().defaultRandom(), + sourceId: t + .uuid() + .notNull() + .references(() => ElectionSource.id, { onDelete: "cascade" }), + electionDate: t.date({ mode: "string" }).notNull(), + county: t.varchar({ length: 100 }).notNull(), + contest: t.text().notNull(), + name: t.text().notNull(), + party: t.varchar({ length: 30 }), + voteFor: t.integer(), + termYears: t.integer(), + hasPrimary: t.boolean(), + isPartisan: t.boolean(), + }), + (table) => ({ + uniqueCandidate: unique().on( + table.sourceId, + table.county, + table.contest, + table.name, + table.party, + ), + lookupIdx: index("election_candidate_lookup_idx").on( + table.electionDate, + table.county, + ), + }), +); + +export const ElectionReferendum = pgTable( + "election_referendum", + (t) => ({ + id: t.uuid().notNull().primaryKey().defaultRandom(), + sourceId: t + .uuid() + .notNull() + .references(() => ElectionSource.id, { onDelete: "cascade" }), + electionDate: t.date({ mode: "string" }).notNull(), + county: t.varchar({ length: 100 }).notNull(), + contest: t.text().notNull(), + choice: t.text().notNull(), + description: t.text(), + }), + (table) => ({ + uniqueChoice: unique().on( + table.sourceId, + table.county, + table.contest, + table.choice, + ), + lookupIdx: index("election_referendum_lookup_idx").on( + table.electionDate, + table.county, + ), + }), +); + +export const ElectionResult = pgTable( + "election_result", + (t) => ({ + id: t.uuid().notNull().primaryKey().defaultRandom(), + sourceId: t + .uuid() + .notNull() + .references(() => ElectionSource.id, { onDelete: "cascade" }), + electionDate: t.date({ mode: "string" }).notNull(), + county: t.varchar({ length: 100 }).notNull(), + precinct: t.varchar({ length: 100 }).notNull(), + contestId: t.varchar({ length: 100 }), + contestType: t.varchar({ length: 30 }), + contest: t.text().notNull(), + choice: t.text().notNull(), + party: t.varchar({ length: 30 }), + voteFor: t.integer(), + electionDayVotes: t.integer().notNull(), + earlyVotingVotes: t.integer().notNull(), + absenteeMailVotes: t.integer().notNull(), + provisionalVotes: t.integer().notNull(), + totalVotes: t.integer().notNull(), + realPrecinct: t.boolean(), + }), + (table) => ({ + uniqueResult: unique().on( + table.sourceId, + table.county, + table.precinct, + table.contest, + table.choice, + ), + lookupIdx: index("election_result_lookup_idx").on( + table.electionDate, + table.county, + ), + }), +); + +// Provider-neutral local-government records populated by scheduled source +// adapters. Provider-specific tables above remain as the Legistar live cache; +// these tables are the durable contract consumed by the product. +export const LocalGovernmentMeeting = pgTable( + "local_government_meeting", + (t) => ({ + id: t.uuid().notNull().primaryKey().defaultRandom(), + source: t.varchar({ length: 50 }).notNull(), + sourceVersion: t.varchar({ length: 100 }).notNull(), + jurisdiction: t.varchar({ length: 100 }).notNull(), + governingBody: t.varchar({ length: 256 }).notNull(), + externalId: t.varchar({ length: 128 }).notNull(), + title: t.text().notNull(), + meetingType: t.varchar({ length: 50 }).notNull(), + status: t.varchar({ length: 50 }).notNull(), + startsAt: t.timestamp({ mode: "date", withTimezone: true }).notNull(), + timezone: t.varchar({ length: 64 }), + location: t.text(), + isCancelled: t.boolean().notNull().default(false), + isAmended: t.boolean().notNull().default(false), + canonicalUrl: t.text().notNull(), + videoUrl: t.text(), + contentHash: t.varchar({ length: 64 }).notNull(), + sourceUpdatedAt: t.timestamp({ mode: "date", withTimezone: true }), + fetchedAt: t + .timestamp({ mode: "date", withTimezone: true }) + .defaultNow() + .notNull(), + createdAt: t.timestamp().defaultNow().notNull(), + updatedAt: t + .timestamp({ mode: "date", withTimezone: true }) + .$onUpdateFn(() => sql`now()`), + }), + (table) => ({ + uniqueSourceMeeting: unique().on( + table.source, + table.jurisdiction, + table.externalId, + ), + jurisdictionDateIdx: index( + "local_government_meeting_jurisdiction_date_idx", + ).on(table.jurisdiction, table.startsAt), + }), +); + +export const LocalGovernmentDocument = pgTable( + "local_government_document", + (t) => ({ + id: t.uuid().notNull().primaryKey().defaultRandom(), + meetingId: t + .uuid() + .notNull() + .references(() => LocalGovernmentMeeting.id, { onDelete: "cascade" }), + type: t.varchar({ length: 30 }).notNull(), + title: t.text().notNull(), + url: t.text().notNull(), + mediaType: t.varchar({ length: 100 }), + checksum: t.varchar({ length: 64 }), + extractedText: t.text(), + isCurrent: t.boolean().notNull().default(true), + discoveredAt: t + .timestamp({ mode: "date", withTimezone: true }) + .defaultNow() + .notNull(), + fetchedAt: t.timestamp({ mode: "date", withTimezone: true }), + createdAt: t.timestamp().defaultNow().notNull(), + updatedAt: t + .timestamp({ mode: "date", withTimezone: true }) + .$onUpdateFn(() => sql`now()`), + }), + (table) => ({ + uniqueMeetingDocument: unique().on(table.meetingId, table.type, table.url), + meetingDocumentIdx: index("local_government_document_meeting_idx").on( + table.meetingId, + ), + }), +); + +export const LocalGovernmentAgendaItem = pgTable( + "local_government_agenda_item", + (t) => ({ + id: t.uuid().notNull().primaryKey().defaultRandom(), + meetingId: t + .uuid() + .notNull() + .references(() => LocalGovernmentMeeting.id, { onDelete: "cascade" }), + externalId: t.varchar({ length: 128 }).notNull(), + sequence: t.integer().notNull(), + itemNumber: t.varchar({ length: 50 }), + section: t.varchar({ length: 100 }), + itemType: t.varchar({ length: 50 }).notNull(), + title: t.text().notNull(), + description: t.text(), + minutesNote: t.text(), + consent: t.boolean().notNull().default(false), + action: t.text(), + motion: t.text(), + outcome: t.varchar({ length: 100 }), + voteSummary: t.text(), + mover: t.varchar({ length: 256 }), + seconder: t.varchar({ length: 256 }), + sourceVersion: t.varchar({ length: 100 }), + contentHash: t.varchar({ length: 64 }), + sourceUpdatedAt: t.timestamp({ mode: "date", withTimezone: true }), + sourceUrl: t.text().notNull(), + createdAt: t.timestamp().defaultNow().notNull(), + updatedAt: t + .timestamp({ mode: "date", withTimezone: true }) + .$onUpdateFn(() => sql`now()`), + }), + (table) => ({ + uniqueMeetingItem: unique().on(table.meetingId, table.externalId), + meetingItemSequenceIdx: index( + "local_government_agenda_item_meeting_sequence_idx", + ).on(table.meetingId, table.sequence), + }), +); + +export const LocalGovernmentVote = pgTable( + "local_government_vote", + (t) => ({ + id: t.uuid().notNull().primaryKey().defaultRandom(), + agendaItemId: t + .uuid() + .notNull() + .references(() => LocalGovernmentAgendaItem.id, { onDelete: "cascade" }), + externalId: t.varchar({ length: 128 }), + voterExternalId: t.varchar({ length: 128 }), + voterName: t.varchar({ length: 256 }).notNull(), + value: t.varchar({ length: 50 }).notNull(), + sort: t.integer().notNull().default(0), + sourceUpdatedAt: t.timestamp({ mode: "date", withTimezone: true }), + fetchedAt: t + .timestamp({ mode: "date", withTimezone: true }) + .defaultNow() + .notNull(), + createdAt: t.timestamp().defaultNow().notNull(), + }), + (table) => ({ + uniqueItemVoter: unique().on(table.agendaItemId, table.voterName), + agendaItemVoteIdx: index("local_government_vote_agenda_item_idx").on( + table.agendaItemId, + ), + }), +); + // Google Civic API response cache export const CivicApiCache = pgTable( "civic_api_cache", diff --git a/packages/env/src/registry.ts b/packages/env/src/registry.ts index 9acc1a6f..472daf54 100644 --- a/packages/env/src/registry.ts +++ b/packages/env/src/registry.ts @@ -85,6 +85,25 @@ const scraperSourceLimitDefinitions = [ ["SCOTUS_MAX_ITEMS", "CourtListener opinion clusters per run.", "50"], ["SCC_CVIG_MAX_ITEMS", "Santa Clara voter-guide PDFs per run.", "10"], ["CA_SOS_MAX_ITEMS", "California SOS office pages per run.", "9"], + ["NCSBE_MAX_ITEMS", "Current-cycle NCSBE source files per run.", "4"], + ["TX_SOS_MAX_ITEMS", "Texas SOS election payloads per run.", "12"], + [ + "TEXAS_LEGISLATURE_MAX_ITEMS", + "Texas Legislature bills per current-session run.", + "100", + ], + [ + "CEDAR_PARK_COUNCIL_MAX_ITEMS", + "Cedar Park City Council meetings per run.", + "100", + ], + ["DURHAM_ONBASE_MAX_ITEMS", "Durham OnBase meetings per run.", "100"], + [ + "DURHAM_ONBASE_CACHE_TTL_HOURS", + "Hours before a Durham OnBase meeting is refreshed.", + "24", + ], + ["DURHAM_BOCC_MAX_ITEMS", "Durham County BOCC meetings per run.", "100"], ] as const; export const envRegistry = [ @@ -374,6 +393,15 @@ export const envRegistry = [ requirements: {}, schema: string, }), + define({ + key: "TEXAS_LEGISLATURE_SESSION", + description: + "Optional current Texas bulk session assertion (for example 892); ingestion rejects historical sessions.", + group: "Scraper sources", + secret: false, + requirements: {}, + schema: string.regex(/^\d{2}(?:R|\d)$/i, "must look like 89R or 892"), + }), define({ key: "BFL_API_KEY", description: "Black Forest Labs key for generated feed-card images.", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c21d57b6..0bedcb04 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -465,6 +465,9 @@ importers: ai: specifier: ^6.0.141 version: 6.0.141(zod@4.3.6) + basic-ftp: + specifier: ^6.0.1 + version: 6.0.1 cheerio: specifier: ^1.2.0 version: 1.2.0 @@ -4668,6 +4671,10 @@ packages: engines: {node: '>=6.0.0'} hasBin: true + basic-ftp@6.0.1: + resolution: {integrity: sha512-3ilxa3n4276wGQp/ImRAuz4ALdsj/2Wd3FqoZBZlajDYnByCZ0JMb4+26Rde0wGXIbM0G2HWSfr/Fi8b21KX8g==} + engines: {node: '>=10.0.0'} + better-auth@1.6.13: resolution: {integrity: sha512-jn8ATnGWDzMwpO4a/3iyW1/RayOF/aoPQOfAeqyCVnQCdqkaONVas9CjbY6PovMsTMa/MG+GRABySfzqtj5J/g==} peerDependencies: @@ -13152,6 +13159,8 @@ snapshots: - '@cloudflare/workers-types' - '@opentelemetry/api' + basic-ftp@6.0.1: {} + better-auth@1.6.13(@opentelemetry/api@1.9.0)(@prisma/client@5.22.0(prisma@5.22.0))(better-sqlite3@12.8.0)(drizzle-kit@0.31.10)(drizzle-orm@0.45.2(@neondatabase/serverless@0.10.0)(@opentelemetry/api@1.9.0)(@planetscale/database@1.19.0)(@prisma/client@5.22.0(prisma@5.22.0))(@types/better-sqlite3@7.6.13)(@types/pg@8.20.0)(@vercel/postgres@0.10.0(utf-8-validate@6.0.4))(better-sqlite3@12.8.0)(gel@2.0.0)(kysely@0.28.17)(mysql2@3.11.3)(pg@8.20.0)(postgres@3.4.4)(prisma@5.22.0))(mysql2@3.11.3)(next@16.2.7(@opentelemetry/api@1.9.0)(@playwright/test@1.58.2)(babel-plugin-react-compiler@1.0.0)(react-dom@19.2.3(react@19.2.3))(react@19.2.3))(pg@8.20.0)(prisma@5.22.0)(react-dom@19.2.3(react@19.2.3))(react@19.2.3): dependencies: '@better-auth/core': 1.6.13(@better-auth/utils@0.4.1)(@better-fetch/fetch@1.1.21)(@opentelemetry/api@1.9.0)(better-call@1.3.5(zod@4.3.6))(jose@6.2.2)(kysely@0.28.17)(nanostores@1.2.0)