From ac89b8aadb14b38e80318a61de04627e87fa0d34 Mon Sep 17 00:00:00 2001 From: ThatXliner Date: Wed, 22 Jul 2026 08:32:06 -0700 Subject: [PATCH] Implement NCSBE current-cycle election data --- apps/scraper/README.md | 9 +- .../src/fixtures/ncsbe/candidates-2026.csv | 3 + .../src/fixtures/ncsbe/referendums-2026.txt | 12 + .../src/fixtures/ncsbe/results-2024.tsv | 3 + .../src/fixtures/ncsbe/results-2026.tsv | 3 + apps/scraper/src/scraper-contracts.ts | 2 + apps/scraper/src/scrapers.ts | 2 + .../src/scrapers/ncsbe-parsers.test.ts | 58 +++ apps/scraper/src/scrapers/ncsbe-parsers.ts | 328 +++++++++++++++ apps/scraper/src/scrapers/ncsbe.config.ts | 11 + apps/scraper/src/scrapers/ncsbe.test.ts | 20 + apps/scraper/src/scrapers/ncsbe.ts | 382 ++++++++++++++++++ docs/ncsbe-election-data.md | 49 +++ packages/api/src/index.ts | 9 + .../api/src/lib/ncsbe-election-data.test.ts | 36 ++ packages/api/src/lib/ncsbe-election-data.ts | 240 +++++++++++ packages/api/src/router/civic.ts | 37 ++ .../db/drizzle/0001_public_election_data.sql | 75 ++++ packages/db/src/schema.ts | 132 ++++++ packages/env/src/registry.ts | 1 + 20 files changed, 1411 insertions(+), 1 deletion(-) create mode 100644 apps/scraper/src/fixtures/ncsbe/candidates-2026.csv create mode 100644 apps/scraper/src/fixtures/ncsbe/referendums-2026.txt create mode 100644 apps/scraper/src/fixtures/ncsbe/results-2024.tsv create mode 100644 apps/scraper/src/fixtures/ncsbe/results-2026.tsv create mode 100644 apps/scraper/src/scrapers/ncsbe-parsers.test.ts create mode 100644 apps/scraper/src/scrapers/ncsbe-parsers.ts create mode 100644 apps/scraper/src/scrapers/ncsbe.config.ts create mode 100644 apps/scraper/src/scrapers/ncsbe.test.ts create mode 100644 apps/scraper/src/scrapers/ncsbe.ts create mode 100644 docs/ncsbe-election-data.md create mode 100644 packages/api/src/lib/ncsbe-election-data.test.ts create mode 100644 packages/api/src/lib/ncsbe-election-data.ts create mode 100644 packages/db/drizzle/0001_public_election_data.sql diff --git a/apps/scraper/README.md b/apps/scraper/README.md index 2e614c78..117e1744 100644 --- a/apps/scraper/README.md +++ b/apps/scraper/README.md @@ -4,7 +4,7 @@ Pulls in government content like bills, court cases, and White House content and ## Active data sources -Only these five are registered and run by `all`: +Only these six are registered and run by `all`: | CLI name | Source and data fetched | Stored/used as | | ------------------- | ------------------------------------------------------------------------------------------ | ----------------------------------------------------------------------------------------------------- | @@ -13,6 +13,7 @@ Only these five are registered and run by `all`: | `scotus` | CourtListener opinion clusters, dockets, and sub-opinion text for the Supreme Court | `court_case`; powers court content and AI/feed enrichment | | `scc-cvig` | Hand-configured Santa Clara County voter-guide PDFs | Candidate statements in `CivicApiCache`; the API matches statements to candidates | | `ca-sos-statements` | California SOS statewide-office candidate-statement pages | Candidate statements in `CivicApiCache`; the API reads the cache and can fall back to the live source | +| `ncsbe` | Current-cycle NCSBE candidate CSV, referendum PDFs, and result ZIPs | Provider-neutral election tables; powers `civic.getNcElectionData` with exact file provenance | `vote411`, `ca-lao-fiscal`, and `ca-vig-archive` remain under `src/scrapers/disabled/` and do not run. Their caches had no application @@ -124,6 +125,7 @@ CONGRESS_MAX_ITEMS=10 pnpm --filter @acme/scraper run start congress | `SCOTUS_MAX_ITEMS` | 50 | CourtListener opinion clusters | | `SCC_CVIG_MAX_ITEMS` | 10 | Voter-guide PDF documents | | `CA_SOS_MAX_ITEMS` | 9 | Statewide-office candidate-statement pages | +| `NCSBE_MAX_ITEMS` | 4 | Current-cycle candidate/referendum/result files | | `SCRAPER_MAX_NEW_ITEMS_PER_RUN` | 10 | New records receiving expensive AI/image enrichment | These are per-run limits, not durable calendar-day quotas. Schedule one run per @@ -132,6 +134,11 @@ invocation gets a fresh allowance. Source limits cap API/page work; `SCRAPER_MAX_NEW_ITEMS_PER_RUN` separately caps expensive enrichment while still storing additional raw records for later backfill. +The NCSBE integration is intentionally current-cycle only and excludes voter +history plus candidate contact/address fields. See +[`docs/ncsbe-election-data.md`](../../docs/ncsbe-election-data.md) for discovery, +idempotency, provenance, API, and deterministic Civic-matching details. + --- ## Congress bills (`congress.ts`) diff --git a/apps/scraper/src/fixtures/ncsbe/candidates-2026.csv b/apps/scraper/src/fixtures/ncsbe/candidates-2026.csv new file mode 100644 index 00000000..cae1dfc3 --- /dev/null +++ b/apps/scraper/src/fixtures/ncsbe/candidates-2026.csv @@ -0,0 +1,3 @@ +"election_dt","county_name","contest_name","name_on_ballot","party_candidate","has_primary","is_partisan","vote_for","term" +"03/03/2026","DURHAM","US HOUSE OF REPRESENTATIVES DISTRICT 04","Nida Allam","DEM","TRUE","TRUE","1","2" +"03/03/2026","WAKE","WAKE COUNTY BOARD OF COMMISSIONERS AT-LARGE","Marguerite Creel","DEM","TRUE","TRUE","2","4" diff --git a/apps/scraper/src/fixtures/ncsbe/referendums-2026.txt b/apps/scraper/src/fixtures/ncsbe/referendums-2026.txt new file mode 100644 index 00000000..f3a816ea --- /dev/null +++ b/apps/scraper/src/fixtures/ncsbe/referendums-2026.txt @@ -0,0 +1,12 @@ +REFERENDUM CHOICES LIST GROUPED BY REFERENDUM +GATES BOARD OF ELECTIONS +CRITERIA: Election: 03/03/2026, County: ALL COUNTIES +GATES +GATES COUNTY LOCAL SALES AND USE TAX REFERENDUM +For Local sales and use tax at the rate of one-quarter percent (0.25%) in addition to all other State and local sales and use taxes. +Against Local sales and use tax at the rate of one-quarter percent (0.25%) in addition to all other State and local sales and use taxes. +GRANVILLE BOARD OF ELECTIONS +GRANVILLE +GRANVILLE COUNTY LOCAL SALES AND USE TAX REFERENDUM +For Local sales and use tax at the rate of one-quarter percent (0.25%) in addition to all other State and local sales and use taxes. +Against Local sales and use tax at the rate of one-quarter percent (0.25%) in addition to all other State and local sales and use taxes. diff --git a/apps/scraper/src/fixtures/ncsbe/results-2024.tsv b/apps/scraper/src/fixtures/ncsbe/results-2024.tsv new file mode 100644 index 00000000..3eb56e2c --- /dev/null +++ b/apps/scraper/src/fixtures/ncsbe/results-2024.tsv @@ -0,0 +1,3 @@ +county election_date precinct contest_group_id contest_type contest_name candidate party vote_for election_day early_voting absentee_by_mail provisional total_votes real_precinct +DURHAM 11/05/2024 01 1373 S US PRESIDENT Kamala D. Harris DEM 1 100 200 30 2 332 Y +WAKE 11/05/2024 01-01 1373 S US PRESIDENT Kamala D. Harris DEM 1 150 250 40 3 443 Y diff --git a/apps/scraper/src/fixtures/ncsbe/results-2026.tsv b/apps/scraper/src/fixtures/ncsbe/results-2026.tsv new file mode 100644 index 00000000..a08290a8 --- /dev/null +++ b/apps/scraper/src/fixtures/ncsbe/results-2026.tsv @@ -0,0 +1,3 @@ +County Election Date Precinct Contest Group ID Contest Type Contest Name Choice Choice Party Vote For Election Day Early Voting Absentee by Mail Provisional Total Votes Real Precinct +DURHAM 03/03/2026 04 2114 S US HOUSE OF REPRESENTATIVES DISTRICT 04 (DEM) Nida Allam DEM 1 371 0 0 0 371 Y +WAKE 03/03/2026 06-05 1 C WAKE COUNTY BOARD OF COMMISSIONERS AT-LARGE (DEM) Marguerite Creel DEM 2 12 0 0 0 12 Y diff --git a/apps/scraper/src/scraper-contracts.ts b/apps/scraper/src/scraper-contracts.ts index bfcfd04f..2a8bd0cf 100644 --- a/apps/scraper/src/scraper-contracts.ts +++ b/apps/scraper/src/scraper-contracts.ts @@ -3,6 +3,7 @@ import type { ScraperEnvContract } from "@acme/env"; import { caSosStatementsConfig } from "./scrapers/ca-sos-statements.config.js"; import { congressConfig } from "./scrapers/congress.config.js"; import { federalregisterConfig } from "./scrapers/federalregister.config.js"; +import { ncsbeConfig } from "./scrapers/ncsbe.config.js"; import { sccCvigConfig } from "./scrapers/scc-cvig.config.js"; import { scotusConfig } from "./scrapers/scotus.config.js"; @@ -12,4 +13,5 @@ export const scraperContracts: readonly ScraperEnvContract[] = [ scotusConfig, sccCvigConfig, caSosStatementsConfig, + ncsbeConfig, ]; diff --git a/apps/scraper/src/scrapers.ts b/apps/scraper/src/scrapers.ts index d4ab8d4d..3f236563 100644 --- a/apps/scraper/src/scrapers.ts +++ b/apps/scraper/src/scrapers.ts @@ -2,6 +2,7 @@ import type { Scraper } from "./utils/types.js"; import { caSosStatements } from "./scrapers/ca-sos-statements.js"; import { congress } from "./scrapers/congress.js"; import { federalregister } from "./scrapers/federalregister.js"; +import { ncsbe } from "./scrapers/ncsbe.js"; import { sccCvig } from "./scrapers/scc-cvig.js"; import { scotus } from "./scrapers/scotus.js"; @@ -11,4 +12,5 @@ export const scrapers: readonly Scraper[] = [ scotus, sccCvig, caSosStatements, + ncsbe, ]; diff --git a/apps/scraper/src/scrapers/ncsbe-parsers.test.ts b/apps/scraper/src/scrapers/ncsbe-parsers.test.ts new file mode 100644 index 00000000..c9abd4d0 --- /dev/null +++ b/apps/scraper/src/scrapers/ncsbe-parsers.test.ts @@ -0,0 +1,58 @@ +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import test from "node:test"; + +import { + parseCandidateCsv, + parseReferendumLines, + parseResultsTsv, + restrictToCycle, +} from "./ncsbe-parsers.js"; + +const fixture = (name: string) => + readFile(new URL(`../fixtures/ncsbe/${name}`, import.meta.url), "utf8"); + +test("parses current-cycle candidate CSV without private contact fields", async () => { + const rows = parseCandidateCsv(await fixture("candidates-2026.csv")); + assert.equal(rows.length, 2); + assert.deepEqual(rows[0], { + electionDate: "2026-03-03", + county: "DURHAM", + contest: "US HOUSE OF REPRESENTATIVES DISTRICT 04", + name: "Nida Allam", + party: "DEM", + voteFor: 1, + termYears: 2, + hasPrimary: true, + isPartisan: true, + }); + assert.equal("email" in rows[0]!, false); + assert.deepEqual( + rows.map((row) => row.county), + ["DURHAM", "WAKE"], + ); +}); + +test("parses NCSBE result layouts from 2026 and 2024 fixtures", async () => { + const current = parseResultsTsv(await fixture("results-2026.tsv")); + const legacy = parseResultsTsv(await fixture("results-2024.tsv")); + assert.equal(current.length, 2); + assert.equal(current[0]?.totalVotes, 371); + assert.equal(current[0]?.earlyVotingVotes, 0); + assert.equal(legacy.length, 2); + assert.equal(legacy[0]?.choice, "Kamala D. Harris"); + assert.deepEqual(restrictToCycle([...current, ...legacy], 2026), current); +}); + +test("parses current-cycle referendum PDF text across counties", async () => { + const rows = parseReferendumLines( + (await fixture("referendums-2026.txt")).split(/\r?\n/), + ); + assert.equal(rows.length, 4); + assert.deepEqual( + [...new Set(rows.map((row) => row.county))], + ["GATES", "GRANVILLE"], + ); + assert.equal(rows[0]?.electionDate, "2026-03-03"); + assert.equal(rows[0]?.choice, "For"); +}); diff --git a/apps/scraper/src/scrapers/ncsbe-parsers.ts b/apps/scraper/src/scrapers/ncsbe-parsers.ts new file mode 100644 index 00000000..bcfa3d76 --- /dev/null +++ b/apps/scraper/src/scrapers/ncsbe-parsers.ts @@ -0,0 +1,328 @@ +import { inflateRawSync } from "node:zlib"; + +export const NCSBE_STRUCTURE_VERSION = "ncsbe-public-election-v1"; + +export interface NcsbeCandidateRecord { + electionDate: string; + county: string; + contest: string; + name: string; + party: string | null; + voteFor: number | null; + termYears: number | null; + hasPrimary: boolean | null; + isPartisan: boolean | null; +} + +export interface NcsbeReferendumRecord { + electionDate: string; + county: string; + contest: string; + choice: string; + description: string | null; +} + +export interface NcsbeResultRecord { + electionDate: string; + county: string; + precinct: string; + contestId: string | null; + contestType: string | null; + contest: string; + choice: string; + party: string | null; + voteFor: number | null; + electionDayVotes: number; + earlyVotingVotes: number; + absenteeMailVotes: number; + provisionalVotes: number; + totalVotes: number; + realPrecinct: boolean | null; +} + +function clean(value: string | undefined): string { + return (value ?? "").replace(/^\uFEFF/, "").trim(); +} + +function nullable(value: string | undefined): string | null { + const result = clean(value); + return result ? result : null; +} + +function integer(value: string | undefined): number | null { + const normalized = clean(value).replace(/,/g, ""); + if (!normalized) return null; + const result = Number(normalized); + return Number.isSafeInteger(result) ? result : null; +} + +function boolean(value: string | undefined): boolean | null { + const normalized = clean(value).toLowerCase(); + if (["true", "t", "yes", "y", "1"].includes(normalized)) return true; + if (["false", "f", "no", "n", "0"].includes(normalized)) return false; + return null; +} + +export function normalizeElectionDate(value: string): string | null { + const normalized = clean(value); + const slash = /^(\d{1,2})\/(\d{1,2})\/(\d{4})$/.exec(normalized); + if (slash) { + return `${slash[3]}-${slash[1]!.padStart(2, "0")}-${slash[2]!.padStart(2, "0")}`; + } + return /^\d{4}-\d{2}-\d{2}$/.test(normalized) ? normalized : null; +} + +/** Small RFC 4180 parser; NCSBE files use commas, quoted fields and CRLF. */ +export function parseDelimited(text: string, delimiter: string): string[][] { + const rows: string[][] = []; + let row: string[] = []; + let field = ""; + let quoted = false; + + for (let i = 0; i < text.length; i++) { + const char = text[i]!; + if (quoted) { + if (char === '"' && text[i + 1] === '"') { + field += '"'; + i++; + } else if (char === '"') { + quoted = false; + } else { + field += char; + } + } else if (char === '"') { + quoted = true; + } else if (char === delimiter) { + row.push(field); + field = ""; + } else if (char === "\n") { + row.push(field.replace(/\r$/, "")); + if (row.some((cell) => cell.length > 0)) rows.push(row); + row = []; + field = ""; + } else { + field += char; + } + } + if (field.length > 0 || row.length > 0) { + row.push(field.replace(/\r$/, "")); + if (row.some((cell) => cell.length > 0)) rows.push(row); + } + return rows; +} + +function records(text: string, delimiter: string): Record[] { + const [rawHeaders, ...rows] = parseDelimited(text, delimiter); + if (!rawHeaders) return []; + const headers = rawHeaders.map((header) => clean(header).toLowerCase()); + return rows.map((row) => + Object.fromEntries( + headers.map((header, index) => [header, row[index] ?? ""]), + ), + ); +} + +function first(row: Record, ...keys: string[]): string { + for (const key of keys) { + if (row[key] !== undefined) return row[key]; + } + return ""; +} + +export function parseCandidateCsv(text: string): NcsbeCandidateRecord[] { + return records(text, ",").flatMap((row) => { + const electionDate = normalizeElectionDate( + first(row, "election_dt", "election_date"), + ); + const county = clean(first(row, "county_name", "county")); + const contest = clean(first(row, "contest_name", "contest")); + const name = clean( + first(row, "name_on_ballot", "candidate_name", "choice"), + ); + if (!electionDate || !county || !contest || !name) return []; + return [ + { + electionDate, + county, + contest, + name, + party: nullable( + first(row, "party_candidate", "candidate_party", "party"), + ), + voteFor: integer(first(row, "vote_for")), + termYears: integer(first(row, "term", "term_years")), + hasPrimary: boolean(first(row, "has_primary")), + isPartisan: boolean(first(row, "is_partisan")), + }, + ]; + }); +} + +export function parseResultsTsv(text: string): NcsbeResultRecord[] { + return records(text, "\t").flatMap((row) => { + const electionDate = normalizeElectionDate( + first(row, "election date", "election_date"), + ); + const county = clean(first(row, "county")); + const precinct = clean(first(row, "precinct")); + const contest = clean(first(row, "contest name", "contest_name")); + const choice = clean(first(row, "choice", "candidate")); + const totals = [ + integer(first(row, "election day", "election_day")), + integer(first(row, "early voting", "early_voting")), + integer(first(row, "absentee by mail", "absentee_by_mail")), + integer(first(row, "provisional")), + integer(first(row, "total votes", "total_votes")), + ]; + if ( + !electionDate || + !county || + !precinct || + !contest || + !choice || + totals.some((n) => n === null) + ) { + return []; + } + return [ + { + electionDate, + county, + precinct, + contestId: nullable(first(row, "contest group id", "contest_group_id")), + contestType: nullable(first(row, "contest type", "contest_type")), + contest, + choice, + party: nullable(first(row, "choice party", "choice_party", "party")), + voteFor: integer(first(row, "vote for", "vote_for")), + electionDayVotes: totals[0]!, + earlyVotingVotes: totals[1]!, + absenteeMailVotes: totals[2]!, + provisionalVotes: totals[3]!, + totalVotes: totals[4]!, + realPrecinct: boolean(first(row, "real precinct", "real_precinct")), + }, + ]; + }); +} + +const NC_COUNTIES = new Set( + `ALAMANCE ALEXANDER ALLEGHANY ANSON ASHE AVERY BEAUFORT BERTIE BLADEN BRUNSWICK BUNCOMBE BURKE CABARRUS CALDWELL CAMDEN CARTERET CASWELL CATAWBA CHATHAM CHEROKEE CHOWAN CLAY CLEVELAND COLUMBUS CRAVEN CUMBERLAND CURRITUCK DARE DAVIDSON DAVIE DUPLIN DURHAM EDGECOMBE FORSYTH FRANKLIN GASTON GATES GRAHAM GRANVILLE GREENE GUILFORD HALIFAX HARNETT HAYWOOD HENDERSON HERTFORD HOKE HYDE IREDELL JACKSON JOHNSTON JONES LEE LENOIR LINCOLN MACON MADISON MARTIN MCDOWELL MECKLENBURG MITCHELL MONTGOMERY MOORE NASH NEW HANOVER NORTHAMPTON ONSLOW ORANGE PAMLICO PASQUOTANK PENDER PERQUIMANS PERSON PITT POLK RANDOLPH RICHMOND ROBESON ROCKINGHAM ROWAN RUTHERFORD SAMPSON SCOTLAND STANLY STOKES SURRY SWAIN TRANSYLVANIA TYRRELL UNION VANCE WAKE WARREN WASHINGTON WATAUGA WAYNE WILKES WILSON YADKIN YANCEY`.split( + " ", + ), +); + +/** Parse position-sorted PDF text lines from NCSBE referendum reports. */ +export function parseReferendumLines( + lines: readonly string[], + sourceElectionDate?: string, +): NcsbeReferendumRecord[] { + let electionDate = sourceElectionDate ?? null; + let county: string | null = null; + let contest: string | null = null; + const output: NcsbeReferendumRecord[] = []; + + for (const raw of lines) { + const line = clean(raw).replace(/\s+/g, " "); + if (!line) continue; + const criteriaDate = /Election:\s*(\d{1,2}\/\d{1,2}\/\d{4})/i.exec( + line, + )?.[1]; + if (criteriaDate) electionDate = normalizeElectionDate(criteriaDate); + const headerCounty = /^([A-Z ]+) BOARD OF ELECTIONS$/ + .exec(line)?.[1] + ?.trim(); + if (headerCounty && NC_COUNTIES.has(headerCounty)) { + county = headerCounty; + contest = null; + continue; + } + if (NC_COUNTIES.has(line)) { + county = line; + contest = null; + continue; + } + if ( + /^(?:REFERENDUM CHOICES|CHOICE DESCRIPTION|CRITERIA:|Page \d+|\w{3} \d{1,2}, \d{4})/i.test( + line, + ) + ) { + continue; + } + const choice = /^(For|Against|Yes|No)\b[\s:.-]*(.*)$/i.exec(line); + if (choice && county && contest && electionDate) { + output.push({ + electionDate, + county, + contest, + choice: + choice[1]![0]!.toUpperCase() + choice[1]!.slice(1).toLowerCase(), + description: nullable(choice[2]), + }); + continue; + } + if (county && /^[A-Z0-9][A-Z0-9 '&().,/%-]+$/.test(line)) contest = line; + } + return output; +} + +/** Read the first text/CSV entry from a conventional NCSBE ZIP archive. */ +export function extractFirstTextFileFromZip(bytes: Uint8Array): string { + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); + let eocd = -1; + for ( + let i = bytes.length - 22; + i >= Math.max(0, bytes.length - 65_557); + i-- + ) { + if (view.getUint32(i, true) === 0x06054b50) { + eocd = i; + break; + } + } + if (eocd < 0) + throw new Error("ZIP end-of-central-directory record not found"); + const entries = view.getUint16(eocd + 10, true); + let offset = view.getUint32(eocd + 16, true); + + for (let entry = 0; entry < entries; entry++) { + if (view.getUint32(offset, true) !== 0x02014b50) + throw new Error("Invalid ZIP central directory"); + const method = view.getUint16(offset + 10, true); + const compressedSize = view.getUint32(offset + 20, true); + const fileNameLength = view.getUint16(offset + 28, true); + const extraLength = view.getUint16(offset + 30, true); + const commentLength = view.getUint16(offset + 32, true); + const localOffset = view.getUint32(offset + 42, true); + const fileName = new TextDecoder().decode( + bytes.subarray(offset + 46, offset + 46 + fileNameLength), + ); + offset += 46 + fileNameLength + extraLength + commentLength; + if (!/\.(?:txt|csv|tsv)$/i.test(fileName)) continue; + if (view.getUint32(localOffset, true) !== 0x04034b50) + throw new Error("Invalid ZIP local header"); + const localNameLength = view.getUint16(localOffset + 26, true); + const localExtraLength = view.getUint16(localOffset + 28, true); + const start = localOffset + 30 + localNameLength + localExtraLength; + const compressed = bytes.subarray(start, start + compressedSize); + const content = + method === 0 + ? compressed + : method === 8 + ? inflateRawSync(compressed) + : null; + if (!content) + throw new Error(`Unsupported ZIP compression method ${method}`); + return new TextDecoder().decode(content); + } + throw new Error("ZIP contains no text election-results file"); +} + +export function restrictToCycle( + rows: readonly T[], + cycleYear: number, +): T[] { + return rows.filter( + (row) => Number(row.electionDate.slice(0, 4)) === cycleYear, + ); +} diff --git a/apps/scraper/src/scrapers/ncsbe.config.ts b/apps/scraper/src/scrapers/ncsbe.config.ts new file mode 100644 index 00000000..4689c441 --- /dev/null +++ b/apps/scraper/src/scrapers/ncsbe.config.ts @@ -0,0 +1,11 @@ +import type { ScraperEnvContract } from "@acme/env"; + +export const ncsbeConfig = { + id: "ncsbe", + name: "North Carolina State Board of Elections", + source: "NCSBE candidate, referendum, and election-results files", + environment: { + required: ["POSTGRES_URL"], + optional: ["NCSBE_MAX_ITEMS"], + }, +} as const satisfies ScraperEnvContract; diff --git a/apps/scraper/src/scrapers/ncsbe.test.ts b/apps/scraper/src/scrapers/ncsbe.test.ts new file mode 100644 index 00000000..c07d555c --- /dev/null +++ b/apps/scraper/src/scrapers/ncsbe.test.ts @@ -0,0 +1,20 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { discoverCurrentCycleFiles } from "./ncsbe.js"; + +test("discovers structured current-cycle files without accepting history", () => { + const candidates = ` + 2026 Candidate List Spreadsheet (CSV) + 2024 Candidate List Spreadsheet (CSV) + 2026 Primary Referendum List`; + const results = ` + 2026 Mar 03 Election - Results (ZIP) + 2024 Nov 05 Election - Results (ZIP)`; + const files = discoverCurrentCycleFiles(candidates, results, 2026); + assert.deepEqual( + files.map((file) => file.kind), + ["candidates", "referenda", "results"], + ); + assert.ok(files.every((file) => file.url.includes("2026"))); +}); diff --git a/apps/scraper/src/scrapers/ncsbe.ts b/apps/scraper/src/scrapers/ncsbe.ts new file mode 100644 index 00000000..544d7d15 --- /dev/null +++ b/apps/scraper/src/scrapers/ncsbe.ts @@ -0,0 +1,382 @@ +/** + * Current-cycle NCSBE public election-data importer. + * + * Runtime discovery is intentionally limited to links for the current calendar + * year. Candidate contact/address fields and all voter-history products are out + * of scope and are never represented in the normalized parser output. + */ +import { createHash } from "node:crypto"; +import { load } from "cheerio"; +import { and, eq, gte, lt, or } from "drizzle-orm"; +import { getDocumentProxy } from "unpdf"; + +import { db } from "@acme/db/client"; +import { + ElectionCandidate, + ElectionReferendum, + ElectionResult, + ElectionSource, +} from "@acme/db/schema"; + +import type { Scraper } from "../utils/types.js"; +import type { + NcsbeCandidateRecord, + NcsbeReferendumRecord, + NcsbeResultRecord, +} from "./ncsbe-parsers.js"; +import { fetchWithRetry } from "../utils/fetch.js"; +import { createLogger } from "../utils/log.js"; +import { + extractFirstTextFileFromZip, + NCSBE_STRUCTURE_VERSION, + normalizeElectionDate, + parseCandidateCsv, + parseReferendumLines, + parseResultsTsv, + restrictToCycle, +} from "./ncsbe-parsers.js"; +import { ncsbeConfig } from "./ncsbe.config.js"; + +const logger = createLogger("ncsbe"); +const CANDIDATE_LIST_URL = "https://www.ncsbe.gov/results-data/candidate-lists"; +const RESULTS_LIST_URL = + "https://www.ncsbe.gov/results-data/election-results/historical-election-results-data"; +const USER_AGENT = "BillionCivicBot/1.0 (+https://billion.app)"; +const BATCH_SIZE = 750; + +type SourceKind = "candidates" | "referenda" | "results"; +type NcsbeRecord = + | NcsbeCandidateRecord + | NcsbeReferendumRecord + | NcsbeResultRecord; + +export interface NcsbeSourceFile { + kind: SourceKind; + url: string; + label: string; +} + +function currentCycleYear(now = new Date()): number { + return now.getUTCFullYear(); +} + +function absoluteUrl(href: string, base: string): string | null { + try { + return new URL(href, base).toString(); + } catch { + return null; + } +} + +function links(html: string, base: string): { label: string; url: string }[] { + const $ = load(html); + return $("a[href]") + .toArray() + .flatMap((anchor) => { + const url = absoluteUrl($(anchor).attr("href") ?? "", base); + return url + ? [{ label: $(anchor).text().replace(/\s+/g, " ").trim(), url }] + : []; + }); +} + +/** Discover only official files whose label/path identifies the current year. */ +export function discoverCurrentCycleFiles( + candidateHtml: string, + resultsHtml: string, + year = currentCycleYear(), +): NcsbeSourceFile[] { + const yearText = String(year); + const discovered: NcsbeSourceFile[] = []; + for (const link of links(candidateHtml, CANDIDATE_LIST_URL)) { + const haystack = `${link.label} ${decodeURIComponent(link.url)}`; + if (!haystack.includes(yearText)) continue; + if (/candidate/i.test(link.label) && /\.csv(?:$|\?)/i.test(link.url)) { + discovered.push({ kind: "candidates", ...link }); + } else if ( + /referendum/i.test(haystack) && + /\.pdf(?:$|\?)/i.test(link.url) + ) { + discovered.push({ kind: "referenda", ...link }); + } + } + for (const link of links(resultsHtml, RESULTS_LIST_URL)) { + const haystack = `${link.label} ${decodeURIComponent(link.url)}`; + if ( + haystack.includes(yearText) && + /results/i.test(link.label) && + /\.zip(?:$|\?)/i.test(link.url) && + !/precinct sort/i.test(link.label) + ) { + discovered.push({ kind: "results", ...link }); + } + } + return [ + ...new Map( + discovered.map((file) => [`${file.kind}:${file.url}`, file]), + ).values(), + ].sort((a, b) => a.kind.localeCompare(b.kind) || a.url.localeCompare(b.url)); +} + +async function fetchBytes(url: string): Promise { + const response = await fetchWithRetry(url, { + headers: { "User-Agent": USER_AGENT }, + maxRetries: 3, + timeoutMs: 45_000, + }); + return new Uint8Array(await response.arrayBuffer()); +} + +async function fetchPage(url: string): Promise { + return new TextDecoder().decode(await fetchBytes(url)); +} + +export async function extractPdfLines(bytes: Uint8Array): Promise { + const pdf = await getDocumentProxy(bytes); + const lines: string[] = []; + for (let pageNumber = 1; pageNumber <= pdf.numPages; pageNumber++) { + const page = await pdf.getPage(pageNumber); + const content = await page.getTextContent(); + const items = (content.items as unknown[]) + .flatMap((raw) => { + const item = raw as { str?: string; transform?: number[] }; + return typeof item.str === "string" && item.transform + ? [ + { + text: item.str.trim(), + x: item.transform[4] ?? 0, + y: item.transform[5] ?? 0, + }, + ] + : []; + }) + .filter((item) => item.text); + items.sort((a, b) => (Math.abs(a.y - b.y) > 2 ? b.y - a.y : a.x - b.x)); + let activeY: number | null = null; + let active: typeof items = []; + const flush = () => { + if (active.length) + lines.push( + active + .sort((a, b) => a.x - b.x) + .map((item) => item.text) + .join(" "), + ); + active = []; + }; + for (const item of items) { + if (activeY !== null && Math.abs(activeY - item.y) > 2) flush(); + activeY = item.y; + active.push(item); + } + flush(); + } + return lines; +} + +export async function parseReferendumPdf( + bytes: Uint8Array, + sourceUrl: string, +): Promise { + return parseReferendumLines( + await extractPdfLines(bytes), + sourceDateFromUrl(sourceUrl) ?? undefined, + ); +} + +function sha256(bytes: Uint8Array): string { + return createHash("sha256").update(bytes).digest("hex"); +} + +function sourceDateFromUrl(url: string): string | null { + const compact = /(?:_|referendums_)(20\d{2})(\d{2})(\d{2})/.exec(url); + return compact + ? normalizeElectionDate(`${compact[2]}/${compact[3]}/${compact[1]}`) + : null; +} + +function groupsByDate( + rows: readonly T[], +): Map { + const groups = new Map(); + for (const row of rows) + groups.set(row.electionDate, [ + ...(groups.get(row.electionDate) ?? []), + row, + ]); + return groups; +} + +async function insertBatches( + rows: readonly T[], + insert: (batch: T[]) => Promise, +): Promise { + for (let start = 0; start < rows.length; start += BATCH_SIZE) { + await insert(rows.slice(start, start + BATCH_SIZE)); + } +} + +async function persistDateGroup( + file: NcsbeSourceFile, + electionDate: string, + checksum: string, + fetchedAt: Date, + rows: NcsbeRecord[], +): Promise { + await db.transaction(async (tx) => { + const [cached] = await tx + .select({ + id: ElectionSource.id, + checksum: ElectionSource.checksum, + structureVersion: ElectionSource.structureVersion, + }) + .from(ElectionSource) + .where( + and( + eq(ElectionSource.provider, "ncsbe"), + eq(ElectionSource.sourceKind, file.kind), + eq(ElectionSource.electionDate, electionDate), + eq(ElectionSource.sourceUrl, file.url), + ), + ) + .limit(1); + if ( + cached?.checksum === checksum && + cached.structureVersion === NCSBE_STRUCTURE_VERSION + ) { + await tx + .update(ElectionSource) + .set({ fetchedAt }) + .where(eq(ElectionSource.id, cached.id)); + return; + } + + const [source] = await tx + .insert(ElectionSource) + .values({ + provider: "ncsbe", + sourceKind: file.kind, + electionDate, + sourceUrl: file.url, + checksum, + structureVersion: NCSBE_STRUCTURE_VERSION, + certificationStatus: + file.kind === "results" ? "official_not_certified" : "not_applicable", + fetchedAt, + }) + .onConflictDoUpdate({ + target: [ + ElectionSource.provider, + ElectionSource.sourceKind, + ElectionSource.electionDate, + ElectionSource.sourceUrl, + ], + set: { checksum, structureVersion: NCSBE_STRUCTURE_VERSION, fetchedAt }, + }) + .returning({ id: ElectionSource.id }); + if (!source) throw new Error(`Unable to upsert source ${file.url}`); + + if (file.kind === "candidates") { + await tx + .delete(ElectionCandidate) + .where(eq(ElectionCandidate.sourceId, source.id)); + const candidates = rows as NcsbeCandidateRecord[]; + await insertBatches(candidates, (batch) => + tx + .insert(ElectionCandidate) + .values(batch.map((row) => ({ ...row, sourceId: source.id }))), + ); + } else if (file.kind === "referenda") { + await tx + .delete(ElectionReferendum) + .where(eq(ElectionReferendum.sourceId, source.id)); + const referenda = rows as NcsbeReferendumRecord[]; + await insertBatches(referenda, (batch) => + tx + .insert(ElectionReferendum) + .values(batch.map((row) => ({ ...row, sourceId: source.id }))), + ); + } else { + await tx + .delete(ElectionResult) + .where(eq(ElectionResult.sourceId, source.id)); + const results = rows as NcsbeResultRecord[]; + await insertBatches(results, (batch) => + tx + .insert(ElectionResult) + .values(batch.map((row) => ({ ...row, sourceId: source.id }))), + ); + } + }); +} + +async function importFile( + file: NcsbeSourceFile, + year: number, +): Promise { + const fetchedAt = new Date(); + const bytes = await fetchBytes(file.url); + const checksum = sha256(bytes); + let rows: NcsbeRecord[]; + if (file.kind === "candidates") { + rows = parseCandidateCsv(new TextDecoder().decode(bytes)); + } else if (file.kind === "results") { + rows = parseResultsTsv(extractFirstTextFileFromZip(bytes)); + } else { + rows = await parseReferendumPdf(bytes, file.url); + } + rows = restrictToCycle(rows, year); + if (rows.length === 0) + throw new Error(`${file.url} produced no current-cycle records`); + for (const [electionDate, dateRows] of groupsByDate(rows)) { + await persistDateGroup(file, electionDate, checksum, fetchedAt, dateRows); + } + return rows.length; +} + +export async function scrapeNcsbe( + maxItems = 4, + now = new Date(), +): Promise { + const year = currentCycleYear(now); + logger.info(`Discovering NCSBE public election files for the ${year} cycle…`); + const [candidateHtml, resultsHtml] = await Promise.all([ + fetchPage(CANDIDATE_LIST_URL), + fetchPage(RESULTS_LIST_URL), + ]); + const files = discoverCurrentCycleFiles( + candidateHtml, + resultsHtml, + year, + ).slice(0, maxItems); + if (files.length === 0) + throw new Error(`No NCSBE files discovered for ${year}`); + for (const file of files) { + const count = await importFile(file, year); + logger.success(`${file.kind}: persisted ${count} records from ${file.url}`); + } + // Keep persistence bounded to the product's current-cycle scope. Cascading + // foreign keys remove normalized rows only after every current import succeeds. + const stale = await db + .delete(ElectionSource) + .where( + and( + eq(ElectionSource.provider, "ncsbe"), + or( + lt(ElectionSource.electionDate, `${year}-01-01`), + gte(ElectionSource.electionDate, `${year + 1}-01-01`), + ), + ), + ) + .returning({ id: ElectionSource.id }); + if (stale.length) + logger.info(`Removed ${stale.length} out-of-cycle NCSBE source snapshots.`); +} + +export const ncsbe: Scraper = { + ...ncsbeConfig, + scrape: (options) => + scrapeNcsbe( + (options?.maxItems ?? Number(process.env.NCSBE_MAX_ITEMS)) || 4, + ), +}; diff --git a/docs/ncsbe-election-data.md b/docs/ncsbe-election-data.md new file mode 100644 index 00000000..287932b8 --- /dev/null +++ b/docs/ncsbe-election-data.md @@ -0,0 +1,49 @@ +# NCSBE current-cycle election data + +The `ncsbe` scraper discovers official public files from the NCSBE candidate-list +and election-results index pages. It does not hard-code election file URLs. At +runtime, discovery accepts only links for the current calendar-year election +cycle and prefers the structured candidate CSV and result ZIP/TSV files. The +official referendum PDF is used because NCSBE does not publish a structured +referendum download. + +```bash +pnpm --filter @acme/scraper run start ncsbe --max-items 4 +``` + +## Data boundaries + +- Candidate addresses, phone numbers, and email addresses are discarded by the + parser and have no database columns. +- Voter registration and voter-history products are never discovered or fetched. +- Historical links may be represented by small parser regression fixtures, but + runtime discovery, persistence, and API reads reject non-current-cycle dates. +- Durham County is the primary validation jurisdiction. Wake County fixtures + prevent county-specific assumptions. + +## Storage and idempotency + +`election_source` stores provider, source kind, exact URL, fetched time, SHA-256, +parser structure version, election date, and certification state. Provider-neutral +candidate, referendum, and result tables reference that row. A re-run upserts the +same source identity and replaces its child rows in one transaction, so interrupted +runs roll back and completed re-runs do not duplicate records. + +The result ZIP's official media file is marked `official_not_certified`. NCSBE's +separate certified-result PDFs are intentionally not used when structured data is +available; a future certified structured file can use `certified` without changing +the reader contract. + +## API reader and Civic matching + +`civic.getNcElectionData` accepts `county`, an ISO `electionDate`, optional Google +Civic contest references, and optional `includePrecincts`. It returns candidate, +referendum, county-total result, and (when requested) precinct records. Every row +includes its exact source URL, checksum, fetched time, structure version, and +certification state. + +Contest and candidate matching first compares normalized values exactly (case, +punctuation, `NC`/`North Carolina`, and primary-party suffixes are normalized). +The only fuzzy fallback is token Dice similarity of at least `0.82`, and it is +accepted only when the best result is at least `0.08` ahead of the runner-up. +Uncertain matches are omitted rather than guessed. diff --git a/packages/api/src/index.ts b/packages/api/src/index.ts index 0d729333..395bd797 100644 --- a/packages/api/src/index.ts +++ b/packages/api/src/index.ts @@ -39,6 +39,15 @@ export type { } from "./lib/elected-officials"; export { getElectedOfficials } from "./lib/elected-officials"; +export type { + CivicContestReference, + ElectionMatch, +} from "./lib/ncsbe-election-data"; +export { + getCurrentNcElectionData, + matchNcsbeName, +} from "./lib/ncsbe-election-data"; + // California Secretary of State live election-results feed export type { ElectionContestResult, diff --git a/packages/api/src/lib/ncsbe-election-data.test.ts b/packages/api/src/lib/ncsbe-election-data.test.ts new file mode 100644 index 00000000..5f0ef86f --- /dev/null +++ b/packages/api/src/lib/ncsbe-election-data.test.ts @@ -0,0 +1,36 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { matchNcsbeName } from "./ncsbe-election-data"; + +void test("matches NCSBE contests exactly after party suffix normalization", () => { + assert.deepEqual( + matchNcsbeName("US SENATE (DEM)", ["US Senate", "NC Senate District 20"]), + { value: "US Senate", method: "exact", score: 1 }, + ); +}); + +void test("uses an unambiguous token fallback for Civic wording differences", () => { + assert.deepEqual( + matchNcsbeName("NC COURT APPEALS JUDGE SEAT 01", [ + "North Carolina Court of Appeals Judge Seat 01", + "North Carolina Supreme Court Associate Justice", + ]), + { + value: "North Carolina Court of Appeals Judge Seat 01", + method: "token", + score: 14 / 15, + }, + ); +}); + +void test("rejects ambiguous or weak fuzzy matches", () => { + assert.equal( + matchNcsbeName("County Board", [ + "County Board Seat 1", + "County Board Seat 2", + ]), + null, + ); + assert.equal(matchNcsbeName("Mayor", ["US Senate"]), null); +}); diff --git a/packages/api/src/lib/ncsbe-election-data.ts b/packages/api/src/lib/ncsbe-election-data.ts new file mode 100644 index 00000000..98a4d652 --- /dev/null +++ b/packages/api/src/lib/ncsbe-election-data.ts @@ -0,0 +1,240 @@ +/** Provider-neutral reader for normalized NCSBE public election records. */ +import { and, eq } from "@acme/db"; +import { db } from "@acme/db/client"; +import { + ElectionCandidate, + ElectionReferendum, + ElectionResult, + ElectionSource, +} from "@acme/db/schema"; + +export interface CivicContestReference { + title: string; + candidates?: string[]; +} + +export type ElectionMatch = { + value: string; + method: "exact" | "token"; + score: number; +} | null; + +/** + * Matching is exact after punctuation/case/party-suffix normalization. The + * documented fallback is token Dice similarity >= .82, accepted only when the + * best candidate is at least .08 ahead of the runner-up to avoid silent ties. + */ +export function matchNcsbeName( + source: string, + targets: readonly string[], +): ElectionMatch { + const normalizedSource = normalize(source); + const exact = targets.find( + (target) => normalize(target) === normalizedSource, + ); + if (exact) return { value: exact, method: "exact", score: 1 }; + const ranked = targets + .map((target) => ({ target, score: dice(tokens(source), tokens(target)) })) + .sort((a, b) => b.score - a.score || a.target.localeCompare(b.target)); + const best = ranked[0]; + const runnerUp = ranked[1]; + return best && + best.score >= 0.82 && + best.score - (runnerUp?.score ?? 0) >= 0.08 + ? { value: best.target, method: "token", score: best.score } + : null; +} + +function normalize(value: string): string { + return value + .toLowerCase() + .replace(/\((?:dem|rep|lib|gre|una|non)\)\s*$/i, "") + .replace(/\bnc\b/g, "north carolina") + .replace(/\b(?:the|office of)\b/g, " ") + .replace(/[^a-z0-9]+/g, " ") + .trim() + .replace(/\s+/g, " "); +} + +function tokens(value: string): Set { + return new Set(normalize(value).split(" ").filter(Boolean)); +} + +function dice(left: Set, right: Set): number { + if (left.size === 0 || right.size === 0) return 0; + let overlap = 0; + for (const token of left) if (right.has(token)) overlap++; + return (2 * overlap) / (left.size + right.size); +} + +function ensureCurrentCycle(electionDate: string, now: Date): void { + const year = Number(electionDate.slice(0, 4)); + if ( + !/^\d{4}-\d{2}-\d{2}$/.test(electionDate) || + year !== now.getUTCFullYear() + ) { + throw new Error("NCSBE reads are limited to the current election cycle"); + } +} + +function contestMatch(contest: string, refs: readonly CivicContestReference[]) { + return matchNcsbeName( + contest, + refs.map((ref) => ref.title), + ); +} + +export async function getCurrentNcElectionData(input: { + county: string; + electionDate: string; + civicContests?: readonly CivicContestReference[]; + includePrecincts?: boolean; + now?: Date; +}) { + ensureCurrentCycle(input.electionDate, input.now ?? new Date()); + const county = input.county + .trim() + .replace(/\s+county$/i, "") + .toUpperCase(); + const sourceFields = { + sourceUrl: ElectionSource.sourceUrl, + fetchedAt: ElectionSource.fetchedAt, + checksum: ElectionSource.checksum, + structureVersion: ElectionSource.structureVersion, + certificationStatus: ElectionSource.certificationStatus, + }; + + const [candidateRows, referendumRows, resultRows] = await Promise.all([ + db + .select({ + electionDate: ElectionCandidate.electionDate, + county: ElectionCandidate.county, + contest: ElectionCandidate.contest, + name: ElectionCandidate.name, + party: ElectionCandidate.party, + voteFor: ElectionCandidate.voteFor, + termYears: ElectionCandidate.termYears, + hasPrimary: ElectionCandidate.hasPrimary, + isPartisan: ElectionCandidate.isPartisan, + ...sourceFields, + }) + .from(ElectionCandidate) + .innerJoin( + ElectionSource, + eq(ElectionCandidate.sourceId, ElectionSource.id), + ) + .where( + and( + eq(ElectionCandidate.electionDate, input.electionDate), + eq(ElectionCandidate.county, county), + ), + ), + db + .select({ + electionDate: ElectionReferendum.electionDate, + county: ElectionReferendum.county, + contest: ElectionReferendum.contest, + choice: ElectionReferendum.choice, + description: ElectionReferendum.description, + ...sourceFields, + }) + .from(ElectionReferendum) + .innerJoin( + ElectionSource, + eq(ElectionReferendum.sourceId, ElectionSource.id), + ) + .where( + and( + eq(ElectionReferendum.electionDate, input.electionDate), + eq(ElectionReferendum.county, county), + ), + ), + db + .select({ + electionDate: ElectionResult.electionDate, + county: ElectionResult.county, + precinct: ElectionResult.precinct, + contestId: ElectionResult.contestId, + contestType: ElectionResult.contestType, + contest: ElectionResult.contest, + choice: ElectionResult.choice, + party: ElectionResult.party, + voteFor: ElectionResult.voteFor, + electionDayVotes: ElectionResult.electionDayVotes, + earlyVotingVotes: ElectionResult.earlyVotingVotes, + absenteeMailVotes: ElectionResult.absenteeMailVotes, + provisionalVotes: ElectionResult.provisionalVotes, + totalVotes: ElectionResult.totalVotes, + realPrecinct: ElectionResult.realPrecinct, + ...sourceFields, + }) + .from(ElectionResult) + .innerJoin(ElectionSource, eq(ElectionResult.sourceId, ElectionSource.id)) + .where( + and( + eq(ElectionResult.electionDate, input.electionDate), + eq(ElectionResult.county, county), + ), + ), + ]); + + const refs = input.civicContests ?? []; + const withContestMatch = ( + rows: T[], + ): (T & { civicMatch: ElectionMatch })[] => { + const matched: (T & { civicMatch: ElectionMatch })[] = []; + for (const row of rows) { + if (refs.length === 0) { + matched.push({ ...row, civicMatch: null }); + continue; + } + const civicMatch = contestMatch(row.contest, refs); + if (civicMatch) matched.push({ ...row, civicMatch }); + } + return matched; + }; + const candidates = withContestMatch(candidateRows).map((row) => { + const ref = refs.find( + (candidate) => candidate.title === row.civicMatch?.value, + ); + return { + ...row, + civicCandidateMatch: ref?.candidates?.length + ? matchNcsbeName(row.name, ref.candidates) + : null, + }; + }); + const referenda = withContestMatch(referendumRows); + const precinctResults = withContestMatch(resultRows); + const totals = new Map< + string, + Omit<(typeof precinctResults)[number], "precinct" | "realPrecinct"> + >(); + for (const row of precinctResults) { + const key = `${row.contest}\u0000${row.choice}\u0000${row.party ?? ""}`; + const existing = totals.get(key); + if (existing) { + existing.electionDayVotes += row.electionDayVotes; + existing.earlyVotingVotes += row.earlyVotingVotes; + existing.absenteeMailVotes += row.absenteeMailVotes; + existing.provisionalVotes += row.provisionalVotes; + existing.totalVotes += row.totalVotes; + } else { + const { + precinct: _precinct, + realPrecinct: _realPrecinct, + ...total + } = row; + totals.set(key, total); + } + } + return { + provider: "ncsbe" as const, + electionDate: input.electionDate, + county, + candidates, + referenda, + results: [...totals.values()], + precinctResults: input.includePrecincts ? precinctResults : undefined, + }; +} diff --git a/packages/api/src/router/civic.ts b/packages/api/src/router/civic.ts index a921b1c6..ff20b7f0 100644 --- a/packages/api/src/router/civic.ts +++ b/packages/api/src/router/civic.ts @@ -10,6 +10,7 @@ import { getVoterInfo, } from "../lib/civic"; import { getElectedOfficials } from "../lib/elected-officials"; +import { getCurrentNcElectionData } from "../lib/ncsbe-election-data"; import { publicProcedure } from "../trpc"; const STATEWIDE_OFFICE = z.enum( @@ -140,4 +141,40 @@ export const civicRouter = { }); } }), + + /** Authoritative current-cycle NC candidates, referenda, and results by county. */ + getNcElectionData: publicProcedure + .input( + z.object({ + county: z.string().trim().min(2).max(100), + electionDate: z.iso.date(), + civicContests: z + .array( + z.object({ + title: z.string().trim().min(1).max(300), + candidates: z + .array(z.string().trim().min(1).max(200)) + .max(50) + .optional(), + }), + ) + .max(100) + .optional(), + includePrecincts: z.boolean().optional(), + }), + ) + .query(async ({ input }) => { + try { + return await getCurrentNcElectionData(input); + } catch (error) { + throw new TRPCError({ + code: "BAD_REQUEST", + message: + error instanceof Error + ? error.message + : "Failed to read NCSBE election data", + cause: error, + }); + } + }), } satisfies TRPCRouterRecord; diff --git a/packages/db/drizzle/0001_public_election_data.sql b/packages/db/drizzle/0001_public_election_data.sql new file mode 100644 index 00000000..8865f4a6 --- /dev/null +++ b/packages/db/drizzle/0001_public_election_data.sql @@ -0,0 +1,75 @@ +CREATE TABLE "election_source" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL, + "provider" varchar(50) NOT NULL, + "source_kind" varchar(30) NOT NULL, + "election_date" date NOT NULL, + "source_url" text NOT NULL, + "checksum" varchar(64) NOT NULL, + "structure_version" varchar(50) NOT NULL, + "certification_status" varchar(30) DEFAULT 'unknown' NOT NULL, + "fetched_at" timestamp with time zone NOT NULL, + "created_at" timestamp DEFAULT now() NOT NULL, + "updated_at" timestamp with time zone, + CONSTRAINT "election_source_provider_source_kind_election_date_source_url_unique" UNIQUE("provider","source_kind","election_date","source_url") +); +--> statement-breakpoint +CREATE TABLE "election_candidate" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL, + "source_id" uuid NOT NULL, + "election_date" date NOT NULL, + "county" varchar(100) NOT NULL, + "contest" text NOT NULL, + "name" text NOT NULL, + "party" varchar(30), + "vote_for" integer, + "term_years" integer, + "has_primary" boolean, + "is_partisan" boolean, + CONSTRAINT "election_candidate_source_id_county_contest_name_party_unique" UNIQUE("source_id","county","contest","name","party") +); +--> statement-breakpoint +CREATE TABLE "election_referendum" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL, + "source_id" uuid NOT NULL, + "election_date" date NOT NULL, + "county" varchar(100) NOT NULL, + "contest" text NOT NULL, + "choice" text NOT NULL, + "description" text, + CONSTRAINT "election_referendum_source_id_county_contest_choice_unique" UNIQUE("source_id","county","contest","choice") +); +--> statement-breakpoint +CREATE TABLE "election_result" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL, + "source_id" uuid NOT NULL, + "election_date" date NOT NULL, + "county" varchar(100) NOT NULL, + "precinct" varchar(100) NOT NULL, + "contest_id" varchar(100), + "contest_type" varchar(30), + "contest" text NOT NULL, + "choice" text NOT NULL, + "party" varchar(30), + "vote_for" integer, + "election_day_votes" integer NOT NULL, + "early_voting_votes" integer NOT NULL, + "absentee_mail_votes" integer NOT NULL, + "provisional_votes" integer NOT NULL, + "total_votes" integer NOT NULL, + "real_precinct" boolean, + CONSTRAINT "election_result_source_id_county_precinct_contest_choice_unique" UNIQUE("source_id","county","precinct","contest","choice") +); +--> statement-breakpoint +ALTER TABLE "election_candidate" ADD CONSTRAINT "election_candidate_source_id_election_source_id_fk" FOREIGN KEY ("source_id") REFERENCES "public"."election_source"("id") ON DELETE cascade ON UPDATE no action; +--> statement-breakpoint +ALTER TABLE "election_referendum" ADD CONSTRAINT "election_referendum_source_id_election_source_id_fk" FOREIGN KEY ("source_id") REFERENCES "public"."election_source"("id") ON DELETE cascade ON UPDATE no action; +--> statement-breakpoint +ALTER TABLE "election_result" ADD CONSTRAINT "election_result_source_id_election_source_id_fk" FOREIGN KEY ("source_id") REFERENCES "public"."election_source"("id") ON DELETE cascade ON UPDATE no action; +--> statement-breakpoint +CREATE INDEX "election_source_date_idx" ON "election_source" USING btree ("election_date"); +--> statement-breakpoint +CREATE INDEX "election_candidate_lookup_idx" ON "election_candidate" USING btree ("election_date","county"); +--> statement-breakpoint +CREATE INDEX "election_referendum_lookup_idx" ON "election_referendum" USING btree ("election_date","county"); +--> statement-breakpoint +CREATE INDEX "election_result_lookup_idx" ON "election_result" USING btree ("election_date","county"); diff --git a/packages/db/src/schema.ts b/packages/db/src/schema.ts index 3c990bae..6aa9d190 100644 --- a/packages/db/src/schema.ts +++ b/packages/db/src/schema.ts @@ -645,6 +645,138 @@ export const LegistarVote = pgTable( }), ); +// Provider-neutral public election data. Source metadata is separated from +// normalized records so every API response can cite the exact upstream file. +// These tables intentionally exclude voter history and candidate contact/address +// fields; election scrapers should persist only public ballot/result facts. +export const ElectionSource = pgTable( + "election_source", + (t) => ({ + id: t.uuid().notNull().primaryKey().defaultRandom(), + provider: t.varchar({ length: 50 }).notNull(), + sourceKind: t.varchar({ length: 30 }).notNull(), + electionDate: t.date({ mode: "string" }).notNull(), + sourceUrl: t.text().notNull(), + checksum: t.varchar({ length: 64 }).notNull(), + structureVersion: t.varchar({ length: 50 }).notNull(), + certificationStatus: t.varchar({ length: 30 }).notNull().default("unknown"), + fetchedAt: t.timestamp({ mode: "date", withTimezone: true }).notNull(), + createdAt: t.timestamp().defaultNow().notNull(), + updatedAt: t + .timestamp({ mode: "date", withTimezone: true }) + .$onUpdateFn(() => sql`now()`), + }), + (table) => ({ + uniqueProviderFile: unique().on( + table.provider, + table.sourceKind, + table.electionDate, + table.sourceUrl, + ), + electionDateIdx: index("election_source_date_idx").on(table.electionDate), + }), +); + +export const ElectionCandidate = pgTable( + "election_candidate", + (t) => ({ + id: t.uuid().notNull().primaryKey().defaultRandom(), + sourceId: t + .uuid() + .notNull() + .references(() => ElectionSource.id, { onDelete: "cascade" }), + electionDate: t.date({ mode: "string" }).notNull(), + county: t.varchar({ length: 100 }).notNull(), + contest: t.text().notNull(), + name: t.text().notNull(), + party: t.varchar({ length: 30 }), + voteFor: t.integer(), + termYears: t.integer(), + hasPrimary: t.boolean(), + isPartisan: t.boolean(), + }), + (table) => ({ + uniqueCandidate: unique().on( + table.sourceId, + table.county, + table.contest, + table.name, + table.party, + ), + lookupIdx: index("election_candidate_lookup_idx").on( + table.electionDate, + table.county, + ), + }), +); + +export const ElectionReferendum = pgTable( + "election_referendum", + (t) => ({ + id: t.uuid().notNull().primaryKey().defaultRandom(), + sourceId: t + .uuid() + .notNull() + .references(() => ElectionSource.id, { onDelete: "cascade" }), + electionDate: t.date({ mode: "string" }).notNull(), + county: t.varchar({ length: 100 }).notNull(), + contest: t.text().notNull(), + choice: t.text().notNull(), + description: t.text(), + }), + (table) => ({ + uniqueChoice: unique().on( + table.sourceId, + table.county, + table.contest, + table.choice, + ), + lookupIdx: index("election_referendum_lookup_idx").on( + table.electionDate, + table.county, + ), + }), +); + +export const ElectionResult = pgTable( + "election_result", + (t) => ({ + id: t.uuid().notNull().primaryKey().defaultRandom(), + sourceId: t + .uuid() + .notNull() + .references(() => ElectionSource.id, { onDelete: "cascade" }), + electionDate: t.date({ mode: "string" }).notNull(), + county: t.varchar({ length: 100 }).notNull(), + precinct: t.varchar({ length: 100 }).notNull(), + contestId: t.varchar({ length: 100 }), + contestType: t.varchar({ length: 30 }), + contest: t.text().notNull(), + choice: t.text().notNull(), + party: t.varchar({ length: 30 }), + voteFor: t.integer(), + electionDayVotes: t.integer().notNull(), + earlyVotingVotes: t.integer().notNull(), + absenteeMailVotes: t.integer().notNull(), + provisionalVotes: t.integer().notNull(), + totalVotes: t.integer().notNull(), + realPrecinct: t.boolean(), + }), + (table) => ({ + uniqueResult: unique().on( + table.sourceId, + table.county, + table.precinct, + table.contest, + table.choice, + ), + lookupIdx: index("election_result_lookup_idx").on( + table.electionDate, + table.county, + ), + }), +); + // Google Civic API response cache export const CivicApiCache = pgTable( "civic_api_cache", diff --git a/packages/env/src/registry.ts b/packages/env/src/registry.ts index 6686f519..5705d0be 100644 --- a/packages/env/src/registry.ts +++ b/packages/env/src/registry.ts @@ -85,6 +85,7 @@ const scraperSourceLimitDefinitions = [ ["SCOTUS_MAX_ITEMS", "CourtListener opinion clusters per run.", "50"], ["SCC_CVIG_MAX_ITEMS", "Santa Clara voter-guide PDFs per run.", "10"], ["CA_SOS_MAX_ITEMS", "California SOS office pages per run.", "9"], + ["NCSBE_MAX_ITEMS", "Current-cycle NCSBE source files per run.", "4"], ] as const; export const envRegistry = [